mithril_cardano_node_internal_database/test/
dummy_cardano_db.rs

1use std::fs::File;
2use std::io::prelude::Write;
3use std::path::{Path, PathBuf};
4
5use mithril_common::entities::{ImmutableFileNumber, SlotNumber};
6
7use crate::entities::{ImmutableFile, LedgerStateSnapshot};
8use crate::{IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR};
9
10/// Dummy cardano db 'immutables' subdirectory content
11struct DummyImmutableDb {
12    /// The dummy cardano db 'immutables' directory path.
13    dir: PathBuf,
14    /// The [immutables files][ImmutableFile] in the dummy cardano db 'immutables' subdirectory.
15    immutables_files: Vec<ImmutableFile>,
16    /// Files that doesn't follow the immutable file name scheme in the dummy cardano db 'immutables' subdirectory.
17    non_immutables_files: Vec<PathBuf>,
18}
19
20/// Dummy cardano db 'ledger' subdirectory content
21struct DummyLedgerStateFolder {
22    /// The dummy cardano db 'ledger' directory path.
23    dir: PathBuf,
24    /// The [ledger state snapshots][LedgerStateSnapshot] in the dummy cardano db 'ledger' subdirectory.
25    ledger_state_snapshots: Vec<LedgerStateSnapshot>,
26    /// Files that are not ledger snapshots in the dummy cardano db 'ledger' subdirectory.
27    non_ledger_state_snapshots: Vec<PathBuf>,
28}
29
30impl DummyImmutableDb {
31    /// Add an immutable chunk file and its primary & secondary to the dummy DB.
32    pub fn add_immutable_file(&mut self) -> ImmutableFileNumber {
33        let new_file_number = self.last_immutable_number().unwrap_or(0) + 1;
34        let mut new_files = write_immutable_trio(None, &self.dir, new_file_number);
35
36        self.immutables_files.append(&mut new_files);
37
38        new_file_number
39    }
40
41    /// Return the file number of the last immutable
42    pub fn last_immutable_number(&self) -> Option<ImmutableFileNumber> {
43        self.immutables_files.last().map(|f| f.number)
44    }
45}
46
47/// A dummy cardano db.
48pub struct DummyCardanoDb {
49    /// The dummy cardano db directory path.
50    dir: PathBuf,
51
52    /// Dummy immutable db
53    immutable_db: DummyImmutableDb,
54
55    /// Dummy ledger state folder
56    ledger_state_folder: DummyLedgerStateFolder,
57}
58
59impl DummyCardanoDb {
60    /// Return the cardano db directory path.
61    pub fn get_dir(&self) -> &PathBuf {
62        &self.dir
63    }
64
65    /// Return the immutable db directory path.
66    pub fn get_immutable_dir(&self) -> &Path {
67        &self.immutable_db.dir
68    }
69
70    /// Return the ledger directory path.
71    pub fn get_ledger_dir(&self) -> &Path {
72        &self.ledger_state_folder.dir
73    }
74
75    /// Return the volatile directory path.
76    pub fn get_volatile_dir(&self) -> PathBuf {
77        self.dir.join(VOLATILE_DIR)
78    }
79
80    /// Return the file number of the last immutable
81    pub fn get_immutable_files(&self) -> &Vec<ImmutableFile> {
82        &self.immutable_db.immutables_files
83    }
84
85    /// Return the path of the files in the ledger directory.
86    pub fn get_ledger_state_snapshots(&self) -> &Vec<LedgerStateSnapshot> {
87        &self.ledger_state_folder.ledger_state_snapshots
88    }
89
90    /// Return the non-ledger state snapshot files in the ledger directory
91    pub fn get_non_ledger_state_snapshots(&self) -> &Vec<PathBuf> {
92        &self.ledger_state_folder.non_ledger_state_snapshots
93    }
94
95    /// Add an immutable chunk file and its primary & secondary to the dummy DB.
96    pub fn add_immutable_file(&mut self) -> ImmutableFileNumber {
97        self.immutable_db.add_immutable_file()
98    }
99
100    /// Return the file number of the last immutable
101    pub fn last_immutable_number(&self) -> Option<ImmutableFileNumber> {
102        self.immutable_db.last_immutable_number()
103    }
104
105    /// Return the non-immutables files in the immutables directory
106    pub fn get_non_immutables_files(&self) -> &Vec<PathBuf> {
107        &self.immutable_db.non_immutables_files
108    }
109}
110
111/// A [DummyCardanoDbBuilder] builder.
112pub struct DummyCardanoDbBuilder {
113    sub_dir: String,
114    immutables_to_write: Vec<ImmutableFileNumber>,
115    non_immutables_to_write: Vec<String>,
116    append_uncompleted_trio: bool,
117    immutable_file_size: Option<u64>,
118    ledger_snapshot_in_memory_to_write: Vec<SlotNumber>,
119    ledger_snapshot_legacy_to_write: Vec<SlotNumber>,
120    non_ledger_snapshot_to_write: Vec<String>,
121    ledger_file_size: Option<u64>,
122    volatile_files_to_write: Vec<String>,
123    volatile_file_size: Option<u64>,
124}
125
126impl DummyCardanoDbBuilder {
127    /// [DummyCardanoDbBuilder] factory, will create a folder with the given `dirname` in the
128    /// system temp directory, if it exists already it will be cleaned.
129    pub fn new(dir_name: &str) -> Self {
130        Self {
131            sub_dir: dir_name.to_string(),
132            immutables_to_write: vec![],
133            non_immutables_to_write: vec![],
134            append_uncompleted_trio: false,
135            immutable_file_size: None,
136            non_ledger_snapshot_to_write: vec![],
137            ledger_snapshot_in_memory_to_write: vec![],
138            ledger_snapshot_legacy_to_write: vec![],
139            ledger_file_size: None,
140            volatile_files_to_write: vec![],
141            volatile_file_size: None,
142        }
143    }
144
145    /// Set the immutables file number that will be used to generate the immutable files, for each
146    /// number three files will be generated (a 'chunk', a 'primary' and a 'secondary' file).
147    pub fn with_immutables(&mut self, immutables: &[ImmutableFileNumber]) -> &mut Self {
148        self.immutables_to_write = immutables.to_vec();
149        self
150    }
151
152    /// Set filenames to write to the db that doesn't follow the immutable file name scheme.
153    pub fn with_non_immutables(&mut self, non_immutables: &[&str]) -> &mut Self {
154        self.non_immutables_to_write = non_immutables.iter().map(|f| f.to_string()).collect();
155        self
156    }
157
158    /// Set legacy ledger state snapshot slot numbers to write to the db in the 'ledger' subdirectory.
159    pub fn with_legacy_ledger_snapshots(&mut self, snapshot_slot_numbers: &[u64]) -> &mut Self {
160        self.ledger_snapshot_legacy_to_write =
161            snapshot_slot_numbers.iter().map(|s| SlotNumber(*s)).collect();
162        self
163    }
164
165    /// Set the slot numbers of utxo-hd in-memory snapshot folders to write to the db in the 'ledger' subdirectory.
166    pub fn with_in_memory_ledger_snapshots(&mut self, snapshot_slot_numbers: &[u64]) -> &mut Self {
167        self.ledger_snapshot_in_memory_to_write =
168            snapshot_slot_numbers.iter().map(|s| SlotNumber(*s)).collect();
169        self
170    }
171
172    /// Set filenames to write to the 'ledger' subdirectory that are not ledger snapshots.
173    pub fn with_non_ledger_files(&mut self, files: &[&str]) -> &mut Self {
174        self.non_ledger_snapshot_to_write = files.iter().map(|name| name.to_string()).collect();
175        self
176    }
177
178    /// Set the size of all ledger files written by [build][Self::build] to the given `file_size` in bytes.
179    ///
180    /// Note: for types of ledger state snapshots that have more than one file, the size is evenly
181    /// split across all files:
182    /// * Legacy snapshots: not divided as it's a single file
183    /// * In-memory snapshots: divided by three
184    pub fn set_ledger_file_size(&mut self, file_size: u64) -> &mut Self {
185        self.ledger_file_size = Some(file_size);
186        self
187    }
188
189    /// Set volatile files to write to the db in the 'volatile' subdirectory.
190    pub fn with_volatile_files(&mut self, files: &[&str]) -> &mut Self {
191        self.volatile_files_to_write = files.iter().map(|f| f.to_string()).collect();
192        self
193    }
194
195    /// Set the size of all volatile files written by [build][Self::build] to the given `file_size` in bytes.
196    pub fn set_volatile_file_size(&mut self, file_size: u64) -> &mut Self {
197        self.volatile_file_size = Some(file_size);
198        self
199    }
200
201    /// Makes [build][Self::build] add another trio of immutables file, that won't be included
202    /// in its returned vec, to simulate the last 3 'uncompleted / wip' files that can be found in
203    /// a cardano immutable db.
204    pub fn append_immutable_trio(&mut self) -> &mut Self {
205        self.append_uncompleted_trio = true;
206        self
207    }
208
209    /// Set the size of all immutable files written by [build][Self::build] to the given `file_size` in bytes.
210    ///
211    /// Note: by default, the size of the produced files is less than 1 kb.
212    pub fn set_immutable_trio_file_size(&mut self, trio_file_size: u64) -> &mut Self {
213        assert!(
214            trio_file_size % 3 == 0,
215            "'trio_file_size' must be a multiple of 3"
216        );
217
218        self.immutable_file_size = Some(trio_file_size / 3);
219        self
220    }
221
222    /// Build a [DummyCardanoDb].
223    pub fn build(&self) -> DummyCardanoDb {
224        let dir = get_test_dir(&self.sub_dir);
225
226        let mut non_immutables_files = vec![];
227        let mut ledger_state_snapshots = vec![];
228        let mut non_ledger_state_snapshots = vec![];
229        let mut immutable_numbers = self.immutables_to_write.clone();
230        immutable_numbers.sort();
231
232        if self.append_uncompleted_trio {
233            write_immutable_trio(
234                self.immutable_file_size,
235                &dir.join(IMMUTABLE_DIR),
236                match immutable_numbers.last() {
237                    None => 0,
238                    Some(last) => last + 1,
239                },
240            );
241        }
242
243        for non_immutable in &self.non_immutables_to_write {
244            non_immutables_files.push(write_dummy_file(
245                self.immutable_file_size,
246                &dir.join(IMMUTABLE_DIR),
247                non_immutable,
248            ));
249        }
250
251        for filename in &self.non_ledger_snapshot_to_write {
252            let ledger_file_path =
253                write_dummy_file(self.ledger_file_size, &dir.join(LEDGER_DIR), filename);
254            non_ledger_state_snapshots.push(ledger_file_path);
255        }
256
257        for slot_number in &self.ledger_snapshot_legacy_to_write {
258            let ledger_file = write_dummy_file(
259                self.ledger_file_size,
260                &dir.join(LEDGER_DIR),
261                &slot_number.to_string(),
262            );
263
264            ledger_state_snapshots.push(LedgerStateSnapshot::legacy(
265                ledger_file,
266                *slot_number,
267                slot_number.to_string().into(),
268            ));
269        }
270
271        for slot_number in &self.ledger_snapshot_in_memory_to_write {
272            let ledger_state_snapshot =
273                write_in_memory_ledger_snapshot(*slot_number, &dir, self.ledger_file_size);
274            ledger_state_snapshots.push(ledger_state_snapshot);
275        }
276
277        for filename in &self.volatile_files_to_write {
278            write_dummy_file(self.volatile_file_size, &dir.join(VOLATILE_DIR), filename);
279        }
280
281        let immutable_db = DummyImmutableDb {
282            dir: dir.join(IMMUTABLE_DIR),
283            immutables_files: immutable_numbers
284                .into_iter()
285                .flat_map(|ifn| {
286                    write_immutable_trio(self.immutable_file_size, &dir.join(IMMUTABLE_DIR), ifn)
287                })
288                .collect::<Vec<_>>(),
289            non_immutables_files,
290        };
291
292        let ledger_state_folder = DummyLedgerStateFolder {
293            dir: dir.join(LEDGER_DIR),
294            ledger_state_snapshots,
295            non_ledger_state_snapshots,
296        };
297
298        DummyCardanoDb {
299            dir,
300            immutable_db,
301            ledger_state_folder,
302        }
303    }
304}
305
306fn write_immutable_trio(
307    optional_size: Option<u64>,
308    dir: &Path,
309    immutable: ImmutableFileNumber,
310) -> Vec<ImmutableFile> {
311    let mut result = vec![];
312    for filename in [
313        format!("{immutable:05}.chunk"),
314        format!("{immutable:05}.primary"),
315        format!("{immutable:05}.secondary"),
316    ] {
317        let file = write_dummy_file(optional_size, dir, &filename);
318        result.push(ImmutableFile {
319            number: immutable.to_owned(),
320            path: file,
321            filename: filename.to_string(),
322        });
323    }
324    result
325}
326
327fn write_in_memory_ledger_snapshot(
328    slot_number: SlotNumber,
329    dir: &Path,
330    optional_size: Option<u64>,
331) -> LedgerStateSnapshot {
332    let ledger_folder_name = slot_number.to_string();
333    let ledger_folder_path = dir.join(LEDGER_DIR).join(&ledger_folder_name);
334    let optional_file_size = optional_size.map(|s| s / 3);
335
336    std::fs::create_dir_all(ledger_folder_path.join(LedgerStateSnapshot::IN_MEMORY_TABLES))
337        .unwrap();
338    write_dummy_file(
339        optional_file_size,
340        &ledger_folder_path,
341        LedgerStateSnapshot::IN_MEMORY_STATE,
342    );
343    write_dummy_file(
344        optional_file_size,
345        &ledger_folder_path,
346        LedgerStateSnapshot::IN_MEMORY_META,
347    );
348    write_dummy_file(
349        optional_file_size,
350        &ledger_folder_path.join(LedgerStateSnapshot::IN_MEMORY_TABLES),
351        LedgerStateSnapshot::IN_MEMORY_TVAR,
352    );
353
354    LedgerStateSnapshot::in_memory(ledger_folder_path, slot_number, ledger_folder_name.into())
355}
356
357/// Create a file with the given name in the given dir, write some text to it, and then
358/// return its path.
359fn write_dummy_file(optional_size: Option<u64>, dir: &Path, filename: &str) -> PathBuf {
360    let file = dir.join(Path::new(filename));
361    let mut source_file = File::create(&file).unwrap();
362
363    write!(source_file, "This is a test file named '{filename}'").unwrap();
364
365    if let Some(file_size) = optional_size {
366        writeln!(source_file).unwrap();
367        source_file.set_len(file_size).unwrap();
368    }
369
370    file
371}
372
373fn get_test_dir(subdir_name: &str) -> PathBuf {
374    // Note: used the common `TestDir` api before, but as the DummyCardanoDb is public, `test_tools` can't be used.
375    let db_dir = std::env::temp_dir()
376        .join("mithril_test")
377        .join("test_cardano_db")
378        .join(subdir_name);
379
380    if db_dir.exists() {
381        std::fs::remove_dir_all(&db_dir)
382            .unwrap_or_else(|e| panic!("Could not remove dir {db_dir:?}: {e}"));
383    }
384    std::fs::create_dir_all(&db_dir)
385        .unwrap_or_else(|e| panic!("Could not create dir {db_dir:?}: {e}"));
386
387    for subdir_name in [LEDGER_DIR, IMMUTABLE_DIR, VOLATILE_DIR] {
388        std::fs::create_dir(db_dir.join(subdir_name)).unwrap();
389    }
390
391    db_dir
392}
393
394#[cfg(test)]
395mod tests {
396    use mithril_common::{assert_dir_eq, current_function};
397
398    use super::*;
399
400    #[test]
401    fn writing_empty_dummy_cardano_db_structure() {
402        let db = DummyCardanoDbBuilder::new(current_function!()).build();
403
404        assert_eq!(db.get_immutable_dir(), db.get_dir().join(IMMUTABLE_DIR));
405        assert_eq!(db.get_ledger_dir(), db.get_dir().join(LEDGER_DIR));
406        assert_eq!(db.get_volatile_dir(), db.get_dir().join(VOLATILE_DIR));
407        assert_eq!(db.last_immutable_number(), None);
408        assert_dir_eq!(
409            &db.dir,
410            format!(
411                "* {IMMUTABLE_DIR}/
412                 * {LEDGER_DIR}/
413                 * {VOLATILE_DIR}/"
414            )
415        );
416    }
417
418    #[test]
419    fn writing_immutable_files() {
420        let db = DummyCardanoDbBuilder::new(current_function!())
421            .with_immutables(&[1, 2])
422            .build();
423
424        assert_eq!(db.last_immutable_number(), Some(2));
425        assert_eq!(db.get_immutable_files().len(), 6); // Each immutable is a trio of files
426        assert_dir_eq!(
427            &db.dir,
428            format!(
429                "* {IMMUTABLE_DIR}/
430                 ** 00001.chunk
431                 ** 00001.primary
432                 ** 00001.secondary
433                 ** 00002.chunk
434                 ** 00002.primary
435                 ** 00002.secondary
436                 * {LEDGER_DIR}/
437                 * {VOLATILE_DIR}/"
438            )
439        );
440    }
441
442    #[test]
443    fn adding_non_completed_immutable_files_trio_is_not_take_in_account_in_resulting_db_metadata() {
444        let db = DummyCardanoDbBuilder::new(current_function!())
445            .with_immutables(&[1])
446            .append_immutable_trio()
447            .build();
448
449        assert_eq!(db.last_immutable_number(), Some(1));
450        assert_eq!(db.get_immutable_files().len(), 3); // Each immutable is a trio of files
451        assert_dir_eq!(
452            &db.dir,
453            format!(
454                "* {IMMUTABLE_DIR}/
455                 ** 00001.chunk
456                 ** 00001.primary
457                 ** 00001.secondary
458                 ** 00002.chunk
459                 ** 00002.primary
460                 ** 00002.secondary
461                 * {LEDGER_DIR}/
462                 * {VOLATILE_DIR}/"
463            )
464        );
465    }
466
467    #[test]
468    fn writing_non_immutable_files() {
469        let db = DummyCardanoDbBuilder::new(current_function!())
470            .with_non_immutables(&["test1.txt", "test2.txt"])
471            .build();
472
473        let non_immutable_files = db.get_non_immutables_files();
474        assert_eq!(
475            non_immutable_files,
476            &vec![
477                db.get_dir().join(IMMUTABLE_DIR).join("test1.txt"),
478                db.get_dir().join(IMMUTABLE_DIR).join("test2.txt"),
479            ]
480        );
481        assert_dir_eq!(
482            &db.dir,
483            format!(
484                "* {IMMUTABLE_DIR}/
485                 ** test1.txt
486                 ** test2.txt
487                 * {LEDGER_DIR}/
488                 * {VOLATILE_DIR}/"
489            )
490        );
491    }
492
493    #[test]
494    fn writing_ledger_snapshots() {
495        let db = DummyCardanoDbBuilder::new(current_function!())
496            .with_legacy_ledger_snapshots(&[100])
497            .with_in_memory_ledger_snapshots(&[200])
498            .build();
499
500        let snapshots = db.get_ledger_state_snapshots();
501        assert_eq!(snapshots.len(), 2);
502        assert!(snapshots.iter().any(|s| s.slot_number() == SlotNumber(100)));
503        assert!(snapshots.iter().any(|s| s.slot_number() == SlotNumber(200)));
504        assert_dir_eq!(
505            &db.dir,
506            format!(
507                "* {IMMUTABLE_DIR}/
508                 * {LEDGER_DIR}/
509                 ** 200/
510                 *** {}/
511                 **** {}
512                 *** {}
513                 *** {}
514                 ** 100
515                 * {VOLATILE_DIR}/",
516                LedgerStateSnapshot::IN_MEMORY_TABLES,
517                LedgerStateSnapshot::IN_MEMORY_TVAR,
518                LedgerStateSnapshot::IN_MEMORY_META,
519                LedgerStateSnapshot::IN_MEMORY_STATE,
520            )
521        );
522    }
523
524    #[test]
525    fn setting_file_sizes() {
526        fn get_file_size(path: &Path) -> u64 {
527            std::fs::metadata(path).unwrap().len()
528        }
529
530        let (immutable_file_size, ledger_file_size, volatile_file_size) = (3000, 6000, 9000);
531        let db = DummyCardanoDbBuilder::new(current_function!())
532            .with_immutables(&[1])
533            .with_legacy_ledger_snapshots(&[100])
534            .with_in_memory_ledger_snapshots(&[200])
535            .with_volatile_files(&["test.txt"])
536            .set_immutable_trio_file_size(immutable_file_size)
537            .set_ledger_file_size(ledger_file_size)
538            .set_volatile_file_size(volatile_file_size)
539            .build();
540
541        for file in db.get_immutable_files() {
542            assert_eq!(get_file_size(&file.path), immutable_file_size / 3);
543        }
544
545        for ledger_state_snapshot in db.get_ledger_state_snapshots() {
546            match ledger_state_snapshot {
547                LedgerStateSnapshot::Legacy { path, .. } => {
548                    assert_eq!(get_file_size(path), ledger_file_size);
549                }
550                LedgerStateSnapshot::InMemory { path, .. } => {
551                    assert_eq!(
552                        get_file_size(&path.join(LedgerStateSnapshot::IN_MEMORY_STATE)),
553                        ledger_file_size / 3
554                    );
555                    assert_eq!(
556                        get_file_size(&path.join(LedgerStateSnapshot::IN_MEMORY_META)),
557                        ledger_file_size / 3
558                    );
559                    assert_eq!(
560                        get_file_size(
561                            &path
562                                .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
563                                .join(LedgerStateSnapshot::IN_MEMORY_TVAR)
564                        ),
565                        ledger_file_size / 3
566                    );
567                }
568            }
569        }
570
571        assert_eq!(
572            get_file_size(&db.get_volatile_dir().join("test.txt")),
573            volatile_file_size
574        );
575    }
576}
mithril_cardano_node_internal_database/test/dummy_cardano_db.rs

mithril_cardano_node_internal_database/test/
dummy_cardano_db.rs
