mithril_common/digesters/
dummy_cardano_db.rs

1use std::fs::File;
2use std::io::prelude::Write;
3use std::path::{Path, PathBuf};
4
5use crate::digesters::LedgerStateSnapshot;
6use crate::entities::SlotNumber;
7use crate::test_utils::TempDir;
8use crate::{
9    digesters::{ImmutableFile, IMMUTABLE_DIR, LEDGER_DIR, VOLATILE_DIR},
10    entities::ImmutableFileNumber,
11};
12
13/// Dummy cardano db 'immutables' subdirectory content
14struct DummyImmutableDb {
15    /// The dummy cardano db 'immutables' directory path.
16    dir: PathBuf,
17    /// The [immutables files][ImmutableFile] in the dummy cardano db 'immutables' subdirectory.
18    immutables_files: Vec<ImmutableFile>,
19    /// Files that doesn't follow the immutable file name scheme in the dummy cardano db 'immutables' subdirectory.
20    non_immutables_files: Vec<PathBuf>,
21}
22
23/// Dummy cardano db 'ledger' subdirectory content
24struct DummyLedgerStateFolder {
25    /// The dummy cardano db 'ledger' directory path.
26    dir: PathBuf,
27    /// The [ledger state snapshots][LedgerStateSnapshot] in the dummy cardano db 'ledger' subdirectory.
28    ledger_state_snapshots: Vec<LedgerStateSnapshot>,
29    /// Files that are not ledger snapshots in the dummy cardano db 'ledger' subdirectory.
30    non_ledger_state_snapshots: Vec<PathBuf>,
31}
32
33impl DummyImmutableDb {
34    /// Add an immutable chunk file and its primary & secondary to the dummy DB.
35    pub fn add_immutable_file(&mut self) -> ImmutableFileNumber {
36        let new_file_number = self.last_immutable_number().unwrap_or(0) + 1;
37        let mut new_files = write_immutable_trio(None, &self.dir, new_file_number);
38
39        self.immutables_files.append(&mut new_files);
40
41        new_file_number
42    }
43
44    /// Return the file number of the last immutable
45    pub fn last_immutable_number(&self) -> Option<ImmutableFileNumber> {
46        self.immutables_files.last().map(|f| f.number)
47    }
48}
49
50/// A dummy cardano db.
51pub struct DummyCardanoDb {
52    /// The dummy cardano db directory path.
53    dir: PathBuf,
54
55    /// Dummy immutable db
56    immutable_db: DummyImmutableDb,
57
58    /// Dummy ledger state folder
59    ledger_state_folder: DummyLedgerStateFolder,
60}
61
62impl DummyCardanoDb {
63    /// Return the cardano db directory path.
64    pub fn get_dir(&self) -> &PathBuf {
65        &self.dir
66    }
67
68    /// Return the immutable db directory path.
69    pub fn get_immutable_dir(&self) -> &Path {
70        &self.immutable_db.dir
71    }
72
73    /// Return the ledger directory path.
74    pub fn get_ledger_dir(&self) -> &Path {
75        &self.ledger_state_folder.dir
76    }
77
78    /// Return the volatile directory path.
79    pub fn get_volatile_dir(&self) -> PathBuf {
80        self.dir.join(VOLATILE_DIR)
81    }
82
83    /// Return the file number of the last immutable
84    pub fn get_immutable_files(&self) -> &Vec<ImmutableFile> {
85        &self.immutable_db.immutables_files
86    }
87
88    /// Return the path of the files in the ledger directory.
89    pub fn get_ledger_state_snapshots(&self) -> &Vec<LedgerStateSnapshot> {
90        &self.ledger_state_folder.ledger_state_snapshots
91    }
92
93    /// Return the non-ledger state snapshot files in the ledger directory
94    pub fn get_non_ledger_state_snapshots(&self) -> &Vec<PathBuf> {
95        &self.ledger_state_folder.non_ledger_state_snapshots
96    }
97
98    /// Add an immutable chunk file and its primary & secondary to the dummy DB.
99    pub fn add_immutable_file(&mut self) -> ImmutableFileNumber {
100        self.immutable_db.add_immutable_file()
101    }
102
103    /// Return the file number of the last immutable
104    pub fn last_immutable_number(&self) -> Option<ImmutableFileNumber> {
105        self.immutable_db.last_immutable_number()
106    }
107
108    /// Return the non-immutables files in the immutables directory
109    pub fn get_non_immutables_files(&self) -> &Vec<PathBuf> {
110        &self.immutable_db.non_immutables_files
111    }
112}
113
114/// A [DummyCardanoDbBuilder] builder.
115pub struct DummyCardanoDbBuilder {
116    sub_dir: String,
117    immutables_to_write: Vec<ImmutableFileNumber>,
118    non_immutables_to_write: Vec<String>,
119    append_uncompleted_trio: bool,
120    immutable_file_size: Option<u64>,
121    ledger_snapshot_in_memory_to_write: Vec<SlotNumber>,
122    ledger_snapshot_legacy_to_write: Vec<SlotNumber>,
123    non_ledger_snapshot_to_write: Vec<String>,
124    ledger_file_size: Option<u64>,
125    volatile_files_to_write: Vec<String>,
126    volatile_file_size: Option<u64>,
127}
128
129impl DummyCardanoDbBuilder {
130    /// [DummyCardanoDbBuilder] factory, will create a folder with the given `dirname` in the
131    /// system temp directory, if it exists already it will be cleaned.
132    pub fn new(dir_name: &str) -> Self {
133        Self {
134            sub_dir: dir_name.to_string(),
135            immutables_to_write: vec![],
136            non_immutables_to_write: vec![],
137            append_uncompleted_trio: false,
138            immutable_file_size: None,
139            non_ledger_snapshot_to_write: vec![],
140            ledger_snapshot_in_memory_to_write: vec![],
141            ledger_snapshot_legacy_to_write: vec![],
142            ledger_file_size: None,
143            volatile_files_to_write: vec![],
144            volatile_file_size: None,
145        }
146    }
147
148    /// Set the immutables file number that will be used to generate the immutable files, for each
149    /// number three files will be generated (a 'chunk', a 'primary' and a 'secondary' file).
150    pub fn with_immutables(&mut self, immutables: &[ImmutableFileNumber]) -> &mut Self {
151        self.immutables_to_write = immutables.to_vec();
152        self
153    }
154
155    /// Set filenames to write to the db that doesn't follow the immutable file name scheme.
156    pub fn with_non_immutables(&mut self, non_immutables: &[&str]) -> &mut Self {
157        self.non_immutables_to_write = non_immutables.iter().map(|f| f.to_string()).collect();
158        self
159    }
160
161    /// Set legacy ledger state snapshot slot numbers to write to the db in the 'ledger' subdirectory.
162    pub fn with_legacy_ledger_snapshots(&mut self, snapshot_slot_numbers: &[u64]) -> &mut Self {
163        self.ledger_snapshot_legacy_to_write = snapshot_slot_numbers
164            .iter()
165            .map(|s| SlotNumber(*s))
166            .collect();
167        self
168    }
169
170    /// Set the slot numbers of utxo-hd in-memory snapshot folders to write to the db in the 'ledger' subdirectory.
171    pub fn with_in_memory_ledger_snapshots(&mut self, snapshot_slot_numbers: &[u64]) -> &mut Self {
172        self.ledger_snapshot_in_memory_to_write = snapshot_slot_numbers
173            .iter()
174            .map(|s| SlotNumber(*s))
175            .collect();
176        self
177    }
178
179    /// Set filenames to write to the 'ledger' subdirectory that are not ledger snapshots.
180    pub fn with_non_ledger_files(&mut self, files: &[&str]) -> &mut Self {
181        self.non_ledger_snapshot_to_write = files.iter().map(|name| name.to_string()).collect();
182        self
183    }
184
185    /// Set the size of all ledger files written by [build][Self::build] to the given `file_size` in bytes.
186    ///
187    /// Note: for types of ledger state snapshots that have more than one file, the size is evenly
188    /// split across all files:
189    /// * Legacy snapshots: not divided as it's a single file
190    /// * In-memory snapshots: divided by three
191    pub fn set_ledger_file_size(&mut self, file_size: u64) -> &mut Self {
192        self.ledger_file_size = Some(file_size);
193        self
194    }
195
196    /// Set volatile files to write to the db in the 'volatile' subdirectory.
197    pub fn with_volatile_files(&mut self, files: &[&str]) -> &mut Self {
198        self.volatile_files_to_write = files.iter().map(|f| f.to_string()).collect();
199        self
200    }
201
202    /// Set the size of all volatile files written by [build][Self::build] to the given `file_size` in bytes.
203    pub fn set_volatile_file_size(&mut self, file_size: u64) -> &mut Self {
204        self.volatile_file_size = Some(file_size);
205        self
206    }
207
208    /// Makes [build][Self::build] add another trio of immutables file, that won't be included
209    /// in its returned vec, to simulate the last 3 'uncompleted / wip' files that can be found in
210    /// a cardano immutable db.
211    pub fn append_immutable_trio(&mut self) -> &mut Self {
212        self.append_uncompleted_trio = true;
213        self
214    }
215
216    /// Set the size of all immutable files written by [build][Self::build] to the given `file_size` in bytes.
217    ///
218    /// Note: by default, the size of the produced files is less than 1 kb.
219    pub fn set_immutable_trio_file_size(&mut self, trio_file_size: u64) -> &mut Self {
220        assert!(
221            trio_file_size % 3 == 0,
222            "'trio_file_size' must be a multiple of 3"
223        );
224
225        self.immutable_file_size = Some(trio_file_size / 3);
226        self
227    }
228
229    /// Build a [DummyCardanoDb].
230    pub fn build(&self) -> DummyCardanoDb {
231        let dir = get_test_dir(&self.sub_dir);
232
233        let mut non_immutables_files = vec![];
234        let mut ledger_state_snapshots = vec![];
235        let mut non_ledger_state_snapshots = vec![];
236        let mut immutable_numbers = self.immutables_to_write.clone();
237        immutable_numbers.sort();
238
239        if self.append_uncompleted_trio {
240            write_immutable_trio(
241                self.immutable_file_size,
242                &dir.join(IMMUTABLE_DIR),
243                match immutable_numbers.last() {
244                    None => 0,
245                    Some(last) => last + 1,
246                },
247            );
248        }
249
250        for non_immutable in &self.non_immutables_to_write {
251            non_immutables_files.push(write_dummy_file(
252                self.immutable_file_size,
253                &dir.join(IMMUTABLE_DIR),
254                non_immutable,
255            ));
256        }
257
258        for filename in &self.non_ledger_snapshot_to_write {
259            let ledger_file_path =
260                write_dummy_file(self.ledger_file_size, &dir.join(LEDGER_DIR), filename);
261            non_ledger_state_snapshots.push(ledger_file_path);
262        }
263
264        for slot_number in &self.ledger_snapshot_legacy_to_write {
265            let ledger_file = write_dummy_file(
266                self.ledger_file_size,
267                &dir.join(LEDGER_DIR),
268                &slot_number.to_string(),
269            );
270
271            ledger_state_snapshots.push(LedgerStateSnapshot::legacy(
272                ledger_file,
273                *slot_number,
274                slot_number.to_string().into(),
275            ));
276        }
277
278        for slot_number in &self.ledger_snapshot_in_memory_to_write {
279            let ledger_state_snapshot =
280                write_in_memory_ledger_snapshot(*slot_number, &dir, self.ledger_file_size);
281            ledger_state_snapshots.push(ledger_state_snapshot);
282        }
283
284        for filename in &self.volatile_files_to_write {
285            write_dummy_file(self.volatile_file_size, &dir.join(VOLATILE_DIR), filename);
286        }
287
288        let immutable_db = DummyImmutableDb {
289            dir: dir.join(IMMUTABLE_DIR),
290            immutables_files: immutable_numbers
291                .into_iter()
292                .flat_map(|ifn| {
293                    write_immutable_trio(self.immutable_file_size, &dir.join(IMMUTABLE_DIR), ifn)
294                })
295                .collect::<Vec<_>>(),
296            non_immutables_files,
297        };
298
299        let ledger_state_folder = DummyLedgerStateFolder {
300            dir: dir.join(LEDGER_DIR),
301            ledger_state_snapshots,
302            non_ledger_state_snapshots,
303        };
304
305        DummyCardanoDb {
306            dir,
307            immutable_db,
308            ledger_state_folder,
309        }
310    }
311}
312
313fn write_immutable_trio(
314    optional_size: Option<u64>,
315    dir: &Path,
316    immutable: ImmutableFileNumber,
317) -> Vec<ImmutableFile> {
318    let mut result = vec![];
319    for filename in [
320        format!("{immutable:05}.chunk"),
321        format!("{immutable:05}.primary"),
322        format!("{immutable:05}.secondary"),
323    ] {
324        let file = write_dummy_file(optional_size, dir, &filename);
325        result.push(ImmutableFile {
326            number: immutable.to_owned(),
327            path: file,
328            filename: filename.to_string(),
329        });
330    }
331    result
332}
333
334fn write_in_memory_ledger_snapshot(
335    slot_number: SlotNumber,
336    dir: &Path,
337    optional_size: Option<u64>,
338) -> LedgerStateSnapshot {
339    let ledger_folder_name = slot_number.to_string();
340    let ledger_folder_path = dir.join(LEDGER_DIR).join(&ledger_folder_name);
341    let optional_file_size = optional_size.map(|s| s / 3);
342
343    std::fs::create_dir_all(ledger_folder_path.join(LedgerStateSnapshot::IN_MEMORY_TABLES))
344        .unwrap();
345    write_dummy_file(
346        optional_file_size,
347        &ledger_folder_path,
348        LedgerStateSnapshot::IN_MEMORY_STATE,
349    );
350    write_dummy_file(
351        optional_file_size,
352        &ledger_folder_path,
353        LedgerStateSnapshot::IN_MEMORY_META,
354    );
355    write_dummy_file(
356        optional_file_size,
357        &ledger_folder_path.join(LedgerStateSnapshot::IN_MEMORY_TABLES),
358        LedgerStateSnapshot::IN_MEMORY_TVAR,
359    );
360
361    LedgerStateSnapshot::in_memory(ledger_folder_path, slot_number, ledger_folder_name.into())
362}
363
364/// Create a file with the given name in the given dir, write some text to it, and then
365/// return its path.
366fn write_dummy_file(optional_size: Option<u64>, dir: &Path, filename: &str) -> PathBuf {
367    let file = dir.join(Path::new(filename));
368    let mut source_file = File::create(&file).unwrap();
369
370    write!(source_file, "This is a test file named '{filename}'").unwrap();
371
372    if let Some(file_size) = optional_size {
373        writeln!(source_file).unwrap();
374        source_file.set_len(file_size).unwrap();
375    }
376
377    file
378}
379
380fn get_test_dir(subdir_name: &str) -> PathBuf {
381    let db_dir = TempDir::create("test_cardano_db", subdir_name);
382    for subdir_name in [LEDGER_DIR, IMMUTABLE_DIR, VOLATILE_DIR] {
383        std::fs::create_dir(db_dir.join(subdir_name)).unwrap();
384    }
385
386    db_dir
387}
388
389#[cfg(test)]
390mod tests {
391    use crate::test_utils::{assert_dir_eq, current_function};
392
393    use super::*;
394
395    #[test]
396    fn writing_empty_dummy_cardano_db_structure() {
397        let db = DummyCardanoDbBuilder::new(current_function!()).build();
398
399        assert_eq!(db.get_immutable_dir(), db.get_dir().join(IMMUTABLE_DIR));
400        assert_eq!(db.get_ledger_dir(), db.get_dir().join(LEDGER_DIR));
401        assert_eq!(db.get_volatile_dir(), db.get_dir().join(VOLATILE_DIR));
402        assert_eq!(db.last_immutable_number(), None);
403        assert_dir_eq!(
404            &db.dir,
405            format!(
406                "* {IMMUTABLE_DIR}/
407                 * {LEDGER_DIR}/
408                 * {VOLATILE_DIR}/"
409            )
410        );
411    }
412
413    #[test]
414    fn writing_immutable_files() {
415        let db = DummyCardanoDbBuilder::new(current_function!())
416            .with_immutables(&[1, 2])
417            .build();
418
419        assert_eq!(db.last_immutable_number(), Some(2));
420        assert_eq!(db.get_immutable_files().len(), 6); // Each immutable is a trio of files
421        assert_dir_eq!(
422            &db.dir,
423            format!(
424                "* {IMMUTABLE_DIR}/
425                 ** 00001.chunk
426                 ** 00001.primary
427                 ** 00001.secondary
428                 ** 00002.chunk
429                 ** 00002.primary
430                 ** 00002.secondary
431                 * {LEDGER_DIR}/
432                 * {VOLATILE_DIR}/"
433            )
434        );
435    }
436
437    #[test]
438    fn adding_non_completed_immutable_files_trio_is_not_take_in_account_in_resulting_db_metadata() {
439        let db = DummyCardanoDbBuilder::new(current_function!())
440            .with_immutables(&[1])
441            .append_immutable_trio()
442            .build();
443
444        assert_eq!(db.last_immutable_number(), Some(1));
445        assert_eq!(db.get_immutable_files().len(), 3); // Each immutable is a trio of files
446        assert_dir_eq!(
447            &db.dir,
448            format!(
449                "* {IMMUTABLE_DIR}/
450                 ** 00001.chunk
451                 ** 00001.primary
452                 ** 00001.secondary
453                 ** 00002.chunk
454                 ** 00002.primary
455                 ** 00002.secondary
456                 * {LEDGER_DIR}/
457                 * {VOLATILE_DIR}/"
458            )
459        );
460    }
461
462    #[test]
463    fn writing_non_immutable_files() {
464        let db = DummyCardanoDbBuilder::new(current_function!())
465            .with_non_immutables(&["test1.txt", "test2.txt"])
466            .build();
467
468        let non_immutable_files = db.get_non_immutables_files();
469        assert_eq!(
470            non_immutable_files,
471            &vec![
472                db.get_dir().join(IMMUTABLE_DIR).join("test1.txt"),
473                db.get_dir().join(IMMUTABLE_DIR).join("test2.txt"),
474            ]
475        );
476        assert_dir_eq!(
477            &db.dir,
478            format!(
479                "* {IMMUTABLE_DIR}/
480                 ** test1.txt
481                 ** test2.txt
482                 * {LEDGER_DIR}/
483                 * {VOLATILE_DIR}/"
484            )
485        );
486    }
487
488    #[test]
489    fn writing_ledger_snapshots() {
490        let db = DummyCardanoDbBuilder::new(current_function!())
491            .with_legacy_ledger_snapshots(&[100])
492            .with_in_memory_ledger_snapshots(&[200])
493            .build();
494
495        let snapshots = db.get_ledger_state_snapshots();
496        assert_eq!(snapshots.len(), 2);
497        assert!(snapshots.iter().any(|s| s.slot_number() == SlotNumber(100)));
498        assert!(snapshots.iter().any(|s| s.slot_number() == SlotNumber(200)));
499        assert_dir_eq!(
500            &db.dir,
501            format!(
502                "* {IMMUTABLE_DIR}/
503                 * {LEDGER_DIR}/
504                 ** 200/
505                 *** {}/
506                 **** {}
507                 *** {}
508                 *** {}
509                 ** 100
510                 * {VOLATILE_DIR}/",
511                LedgerStateSnapshot::IN_MEMORY_TABLES,
512                LedgerStateSnapshot::IN_MEMORY_TVAR,
513                LedgerStateSnapshot::IN_MEMORY_META,
514                LedgerStateSnapshot::IN_MEMORY_STATE,
515            )
516        );
517    }
518
519    #[test]
520    fn setting_file_sizes() {
521        fn get_file_size(path: &Path) -> u64 {
522            std::fs::metadata(path).unwrap().len()
523        }
524
525        let (immutable_file_size, ledger_file_size, volatile_file_size) = (3000, 6000, 9000);
526        let db = DummyCardanoDbBuilder::new(current_function!())
527            .with_immutables(&[1])
528            .with_legacy_ledger_snapshots(&[100])
529            .with_in_memory_ledger_snapshots(&[200])
530            .with_volatile_files(&["test.txt"])
531            .set_immutable_trio_file_size(immutable_file_size)
532            .set_ledger_file_size(ledger_file_size)
533            .set_volatile_file_size(volatile_file_size)
534            .build();
535
536        for file in db.get_immutable_files() {
537            assert_eq!(get_file_size(&file.path), immutable_file_size / 3);
538        }
539
540        for ledger_state_snapshot in db.get_ledger_state_snapshots() {
541            match ledger_state_snapshot {
542                LedgerStateSnapshot::Legacy { path, .. } => {
543                    assert_eq!(get_file_size(path), ledger_file_size);
544                }
545                LedgerStateSnapshot::InMemory { path, .. } => {
546                    assert_eq!(
547                        get_file_size(&path.join(LedgerStateSnapshot::IN_MEMORY_STATE)),
548                        ledger_file_size / 3
549                    );
550                    assert_eq!(
551                        get_file_size(&path.join(LedgerStateSnapshot::IN_MEMORY_META)),
552                        ledger_file_size / 3
553                    );
554                    assert_eq!(
555                        get_file_size(
556                            &path
557                                .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
558                                .join(LedgerStateSnapshot::IN_MEMORY_TVAR)
559                        ),
560                        ledger_file_size / 3
561                    );
562                }
563            }
564        }
565
566        assert_eq!(
567            get_file_size(&db.get_volatile_dir().join("test.txt")),
568            volatile_file_size
569        );
570    }
571}
mithril_common/digesters/dummy_cardano_db.rs

mithril_common/digesters/
dummy_cardano_db.rs
