mithril_aggregator/services/snapshotter/
compressed_archive_snapshotter.rs

1use anyhow::{anyhow, Context};
2use async_trait::async_trait;
3use slog::{debug, warn, Logger};
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use mithril_cardano_node_internal_database::entities::AncillaryFilesManifest;
9use mithril_cardano_node_internal_database::entities::{ImmutableFile, LedgerStateSnapshot};
10use mithril_cardano_node_internal_database::{immutable_trio_names, IMMUTABLE_DIR, LEDGER_DIR};
11use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber};
12use mithril_common::logging::LoggerExtensions;
13use mithril_common::StdResult;
14
15use crate::dependency_injection::DependenciesBuilderError;
16use crate::tools::file_archiver::appender::{AppenderData, AppenderEntries, TarAppender};
17use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver};
18use crate::tools::file_size;
19
20use super::{ancillary_signer::AncillarySigner, Snapshotter};
21
22/// Compressed Archive Snapshotter create a compressed file.
23pub struct CompressedArchiveSnapshotter {
24    /// DB directory to snapshot
25    db_directory: PathBuf,
26
27    /// Directory to store ongoing snapshot
28    ongoing_snapshot_directory: PathBuf,
29
30    /// Compression algorithm to use
31    compression_algorithm: CompressionAlgorithm,
32
33    file_archiver: Arc<FileArchiver>,
34
35    ancillary_signer: Arc<dyn AncillarySigner>,
36
37    logger: Logger,
38}
39
40#[async_trait]
41impl Snapshotter for CompressedArchiveSnapshotter {
42    async fn snapshot_all_completed_immutables(
43        &self,
44        archive_name_without_extension: &str,
45    ) -> StdResult<FileArchive> {
46        debug!(
47            self.logger,
48            "Snapshotting all completed immutables into archive: '{archive_name_without_extension}'"
49        );
50
51        let paths_to_include = ImmutableFile::list_completed_in_dir(&self.db_directory)
52            .with_context(|| {
53                format!(
54                    "Can not list completed immutables in database directory: '{}'",
55                    self.db_directory.display()
56                )
57            })?
58            .into_iter()
59            .map(|immutable_file: ImmutableFile| {
60                PathBuf::from(IMMUTABLE_DIR).join(immutable_file.filename)
61            })
62            .collect();
63        let appender = AppenderEntries::new(paths_to_include, self.db_directory.clone())?;
64        self.snapshot(archive_name_without_extension, appender)
65            .await
66    }
67
68    async fn snapshot_ancillary(
69        &self,
70        immutable_file_number: ImmutableFileNumber,
71        archive_name_without_extension: &str,
72    ) -> StdResult<FileArchive> {
73        debug!(
74            self.logger,
75            "Snapshotting ancillary archive: '{archive_name_without_extension}'"
76        );
77
78        let temp_snapshot_directory =
79            self.temp_ancillary_snapshot_directory(archive_name_without_extension);
80        fs::create_dir(&temp_snapshot_directory).with_context(|| {
81            format!(
82                "Can not create temporary snapshot ancillary directory: '{}'",
83                temp_snapshot_directory.display()
84            )
85        })?;
86
87        let snapshot_result = self
88            .snapshot_ancillary_from_temp_directory(
89                immutable_file_number,
90                &temp_snapshot_directory,
91                archive_name_without_extension,
92            )
93            .await;
94
95        if let Err(e) = fs::remove_dir_all(&temp_snapshot_directory) {
96            warn!(
97                self.logger, "Failed to remove temporary snapshot ancillary directory '{}'", temp_snapshot_directory.display();
98                "error" => ?e
99            );
100        }
101        snapshot_result
102    }
103
104    async fn snapshot_immutable_trio(
105        &self,
106        immutable_file_number: ImmutableFileNumber,
107        archive_name_without_extension: &str,
108    ) -> StdResult<FileArchive> {
109        debug!(
110            self.logger,
111            "Snapshotting immutable trio {immutable_file_number} into archive '{archive_name_without_extension}'"
112        );
113
114        let files_to_archive = immutable_trio_names(immutable_file_number)
115            .iter()
116            .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
117            .collect();
118        let appender = AppenderEntries::new(files_to_archive, self.db_directory.clone())?;
119
120        self.snapshot(archive_name_without_extension, appender)
121            .await
122    }
123
124    async fn compute_immutable_files_total_uncompressed_size(
125        &self,
126        up_to_immutable_file_number: ImmutableFileNumber,
127    ) -> StdResult<u64> {
128        if up_to_immutable_file_number == 0 {
129            return Err(anyhow!(
130                "Could not compute the total size without immutable files"
131            ));
132        }
133        let immutable_directory = self.db_directory.join(IMMUTABLE_DIR);
134
135        tokio::task::spawn_blocking(move || -> StdResult<_> {
136            let immutable_paths = (1..=up_to_immutable_file_number)
137                .flat_map(immutable_trio_names)
138                .map(|filename| immutable_directory.join(filename))
139                .collect();
140
141            file_size::compute_size(immutable_paths)
142        })
143        .await?
144    }
145
146    fn compression_algorithm(&self) -> CompressionAlgorithm {
147        self.compression_algorithm
148    }
149}
150
151impl CompressedArchiveSnapshotter {
152    /// Snapshotter factory
153    pub fn new(
154        db_directory: PathBuf,
155        ongoing_snapshot_directory: PathBuf,
156        compression_algorithm: CompressionAlgorithm,
157        file_archiver: Arc<FileArchiver>,
158        ancillary_signer: Arc<dyn AncillarySigner>,
159        logger: Logger,
160    ) -> StdResult<CompressedArchiveSnapshotter> {
161        if ongoing_snapshot_directory.exists() {
162            fs::remove_dir_all(&ongoing_snapshot_directory).with_context(|| {
163                format!(
164                    "Can not remove snapshotter directory: '{}'.",
165                    ongoing_snapshot_directory.display()
166                )
167            })?;
168        }
169
170        fs::create_dir(&ongoing_snapshot_directory).map_err(|e| {
171            DependenciesBuilderError::Initialization {
172                message: format!(
173                    "Can not create snapshotter directory: '{}'.",
174                    ongoing_snapshot_directory.display()
175                ),
176                error: Some(e.into()),
177            }
178        })?;
179
180        Ok(Self {
181            db_directory,
182            ongoing_snapshot_directory,
183            compression_algorithm,
184            file_archiver,
185            ancillary_signer,
186            logger: logger.new_with_component_name::<Self>(),
187        })
188    }
189
190    async fn snapshot<T: TarAppender + 'static>(
191        &self,
192        name_without_extension: &str,
193        appender: T,
194    ) -> StdResult<FileArchive> {
195        let file_archiver = self.file_archiver.clone();
196        let parameters = ArchiveParameters {
197            archive_name_without_extension: name_without_extension.to_string(),
198            target_directory: self.ongoing_snapshot_directory.clone(),
199            compression_algorithm: self.compression_algorithm,
200        };
201
202        // spawn a separate thread to prevent blocking
203        let file_archive = tokio::task::spawn_blocking(move || -> StdResult<FileArchive> {
204            file_archiver.archive(parameters, appender)
205        })
206        .await??;
207
208        Ok(file_archive)
209    }
210
211    fn temp_ancillary_snapshot_directory(&self, discriminator: &str) -> PathBuf {
212        self.ongoing_snapshot_directory
213            .join(format!("temp-ancillary-{discriminator}"))
214    }
215
216    async fn snapshot_ancillary_from_temp_directory(
217        &self,
218        immutable_file_number: ImmutableFileNumber,
219        temp_snapshot_directory: &Path,
220        archive_name_without_extension: &str,
221    ) -> StdResult<FileArchive> {
222        let paths_to_include = self
223            .get_files_and_directories_for_ancillary_snapshot(
224                immutable_file_number,
225                temp_snapshot_directory,
226            )
227            .await?;
228        let signed_manifest = self
229            .build_and_sign_ancillary_manifest(paths_to_include.clone(), temp_snapshot_directory)
230            .await?;
231        let appender =
232            AppenderEntries::new(paths_to_include, temp_snapshot_directory.to_path_buf())?.chain(
233                AppenderData::from_json(
234                    PathBuf::from(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME),
235                    &signed_manifest,
236                )?,
237            );
238        self.snapshot(archive_name_without_extension, appender)
239            .await
240    }
241
242    /// Returns the list of files and directories to include in ancillary snapshot.
243    ///
244    /// Those files will be copied to the target folder in order to have fixed copies that do not
245    /// change during the snapshot process.
246    ///
247    /// The immutable file included in the ancillary archive corresponds to the last one (and not finalized yet)
248    /// when the immutable file number given to the function corresponds to the penultimate.
249    async fn get_files_and_directories_for_ancillary_snapshot(
250        &self,
251        immutable_file_number: u64,
252        target_folder: &Path,
253    ) -> StdResult<Vec<PathBuf>> {
254        let next_immutable_file_number = immutable_file_number + 1;
255        let mut files_to_snapshot: Vec<PathBuf> = immutable_trio_names(next_immutable_file_number)
256            .into_iter()
257            .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
258            .collect();
259
260        let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
261        let ledger_files = LedgerStateSnapshot::list_all_in_dir(&db_ledger_dir)?;
262        let latest_ledger_files: Vec<PathBuf> = ledger_files
263            .iter()
264            .rev()
265            .take(2)
266            .flat_map(|ledger_state_snapshot| ledger_state_snapshot.get_files_relative_path())
267            .map(|path| PathBuf::from(LEDGER_DIR).join(path))
268            .collect();
269        if latest_ledger_files.is_empty() {
270            return Err(anyhow!(
271                "No ledger state snapshot found in the ledger directory: '{}'",
272                db_ledger_dir.display()
273            ));
274        }
275        files_to_snapshot.extend(latest_ledger_files);
276
277        fs::create_dir(target_folder.join(IMMUTABLE_DIR))
278            .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
279        fs::create_dir(target_folder.join(LEDGER_DIR))
280            .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
281
282        for file in &files_to_snapshot {
283            // Some files to snapshot are in subfolders (i.e.: in-memory ledger snapshots files)
284            if let Some(parent_dir) = file.parent() {
285                let target_parent_dir = target_folder.join(parent_dir);
286                if !target_parent_dir.exists() {
287                    fs::create_dir_all(&target_parent_dir).with_context(|| {
288                        format!("Can not create folder: `{}`", target_parent_dir.display())
289                    })?;
290                }
291            }
292
293            let source = self.db_directory.join(file);
294            let target = target_folder.join(file);
295            tokio::fs::copy(&source, &target).await.with_context(|| {
296                format!(
297                    "Failed to copy file `{}` to `{}`",
298                    source.display(),
299                    target.display()
300                )
301            })?;
302        }
303
304        Ok(files_to_snapshot)
305    }
306
307    async fn build_and_sign_ancillary_manifest(
308        &self,
309        paths_to_include: Vec<PathBuf>,
310        temp_snapshot_directory: &Path,
311    ) -> StdResult<AncillaryFilesManifest> {
312        let mut manifest =
313            AncillaryFilesManifest::from_paths(temp_snapshot_directory, paths_to_include).await?;
314        let signature = self
315            .ancillary_signer
316            .compute_ancillary_manifest_signature(&manifest)
317            .await?;
318        manifest.set_signature(signature);
319
320        Ok(manifest)
321    }
322}
323
324#[cfg(test)]
325mod tests {
326    use std::fs::File;
327    use std::path::Path;
328    use std::sync::Arc;
329
330    use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder;
331    use mithril_common::test_utils::assert_equivalent;
332    use mithril_common::{assert_dir_eq, current_function, temp_dir_create};
333
334    use crate::services::ancillary_signer::MockAncillarySigner;
335    use crate::test_tools::TestLogger;
336
337    use super::*;
338
339    fn list_files(test_dir: &Path) -> Vec<String> {
340        fs::read_dir(test_dir)
341            .unwrap()
342            .map(|f| f.unwrap().file_name().to_str().unwrap().to_owned())
343            .collect()
344    }
345
346    fn snapshotter_for_test(
347        test_directory: &Path,
348        db_directory: &Path,
349        compression_algorithm: CompressionAlgorithm,
350    ) -> CompressedArchiveSnapshotter {
351        CompressedArchiveSnapshotter::new(
352            db_directory.to_path_buf(),
353            test_directory.join("ongoing_snapshot"),
354            compression_algorithm,
355            Arc::new(FileArchiver::new_for_test(
356                test_directory.join("verification"),
357            )),
358            Arc::new(MockAncillarySigner::new()),
359            TestLogger::stdout(),
360        )
361        .unwrap()
362    }
363
364    #[test]
365    fn return_parametrized_compression_algorithm() {
366        let test_dir = temp_dir_create!();
367        let snapshotter = snapshotter_for_test(
368            &test_dir,
369            Path::new("whatever"),
370            CompressionAlgorithm::Zstandard,
371        );
372
373        assert_eq!(
374            CompressionAlgorithm::Zstandard,
375            snapshotter.compression_algorithm()
376        );
377    }
378
379    #[test]
380    fn should_create_directory_if_does_not_exist() {
381        let test_dir = temp_dir_create!();
382        let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
383        let db_directory = test_dir.join("whatever");
384
385        CompressedArchiveSnapshotter::new(
386            db_directory,
387            ongoing_snapshot_directory.clone(),
388            CompressionAlgorithm::Gzip,
389            Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
390            Arc::new(MockAncillarySigner::new()),
391            TestLogger::stdout(),
392        )
393        .unwrap();
394
395        assert!(ongoing_snapshot_directory.is_dir());
396    }
397
398    #[test]
399    fn should_clean_ongoing_snapshot_directory_if_already_exists() {
400        let test_dir = temp_dir_create!();
401        let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
402        let db_directory = test_dir.join("whatever");
403
404        fs::create_dir_all(&ongoing_snapshot_directory).unwrap();
405
406        File::create(ongoing_snapshot_directory.join("whatever.txt")).unwrap();
407
408        CompressedArchiveSnapshotter::new(
409            db_directory,
410            ongoing_snapshot_directory.clone(),
411            CompressionAlgorithm::Gzip,
412            Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
413            Arc::new(MockAncillarySigner::new()),
414            TestLogger::stdout(),
415        )
416        .unwrap();
417
418        assert_eq!(0, fs::read_dir(ongoing_snapshot_directory).unwrap().count());
419    }
420
421    #[tokio::test]
422    async fn should_create_snapshots_in_its_ongoing_snapshot_directory() {
423        let test_dir = temp_dir_create!();
424        let pending_snapshot_directory = test_dir.join("pending_snapshot");
425        let cardano_db = DummyCardanoDbBuilder::new(current_function!())
426            .with_immutables(&[1])
427            .append_immutable_trio()
428            .build();
429
430        let snapshotter = CompressedArchiveSnapshotter::new(
431            cardano_db.get_dir().clone(),
432            pending_snapshot_directory.clone(),
433            CompressionAlgorithm::Gzip,
434            Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
435            Arc::new(MockAncillarySigner::new()),
436            TestLogger::stdout(),
437        )
438        .unwrap();
439        let snapshot = snapshotter
440            .snapshot_all_completed_immutables("whatever")
441            .await
442            .unwrap();
443
444        assert_eq!(
445            pending_snapshot_directory,
446            snapshot.get_file_path().parent().unwrap()
447        );
448    }
449
450    mod snapshot_all_completed_immutables {
451        use super::*;
452
453        #[tokio::test]
454        async fn include_only_completed_immutables() {
455            let test_dir = temp_dir_create!();
456            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
457                .with_immutables(&[1, 2, 3])
458                .append_immutable_trio()
459                .with_legacy_ledger_snapshots(&[437])
460                .with_volatile_files(&["blocks-0.dat"])
461                .with_non_immutables(&["random_file.txt", "00002.trap"])
462                .build();
463
464            let snapshotter =
465                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
466
467            let snapshot = snapshotter
468                .snapshot_all_completed_immutables("completed_immutables")
469                .await
470                .unwrap();
471
472            let unpack_dir = snapshot.unpack_gzip(&test_dir);
473            let unpacked_files = list_files(&unpack_dir);
474            let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
475
476            assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
477            assert_equivalent(
478                vec![
479                    "00001.chunk".to_string(),
480                    "00001.primary".to_string(),
481                    "00001.secondary".to_string(),
482                    "00002.chunk".to_string(),
483                    "00002.primary".to_string(),
484                    "00002.secondary".to_string(),
485                    "00003.chunk".to_string(),
486                    "00003.primary".to_string(),
487                    "00003.secondary".to_string(),
488                ],
489                unpacked_immutable_files,
490            );
491        }
492    }
493
494    mod snapshot_immutable_trio {
495        use super::*;
496
497        #[tokio::test]
498        async fn include_only_immutable_trio() {
499            let test_dir = temp_dir_create!();
500            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
501                .with_immutables(&[1, 2, 3])
502                .with_legacy_ledger_snapshots(&[437])
503                .with_volatile_files(&["blocks-0.dat"])
504                .with_non_immutables(&["random_file.txt", "00002.trap"])
505                .build();
506
507            let snapshotter =
508                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
509
510            let snapshot = snapshotter
511                .snapshot_immutable_trio(2, "immutable-2")
512                .await
513                .unwrap();
514
515            let unpack_dir = snapshot.unpack_gzip(&test_dir);
516            let unpacked_files = list_files(&unpack_dir);
517            let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
518
519            assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
520            assert_equivalent(
521                vec![
522                    "00002.chunk".to_string(),
523                    "00002.primary".to_string(),
524                    "00002.secondary".to_string(),
525                ],
526                unpacked_immutable_files,
527            );
528        }
529    }
530
531    mod snapshot_ancillary {
532        use mithril_common::test_utils::fake_keys;
533
534        use super::*;
535
536        #[tokio::test]
537        async fn getting_files_to_include_for_legacy_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
538        ) {
539            let test_dir = temp_dir_create!();
540            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
541                .with_immutables(&[1, 2])
542                .with_legacy_ledger_snapshots(&[737])
543                .build();
544            let snapshotter =
545                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
546            let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
547            fs::create_dir(&ancillary_snapshot_dir).unwrap();
548
549            snapshotter
550                .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
551                .await
552                .unwrap();
553
554            assert_dir_eq!(
555                &ancillary_snapshot_dir,
556                format!(
557                    "* {IMMUTABLE_DIR}/
558                     ** 00002.chunk
559                     ** 00002.primary
560                     ** 00002.secondary
561                     * {LEDGER_DIR}/
562                     ** 737"
563                )
564            );
565        }
566
567        #[tokio::test]
568        async fn getting_files_to_include_for_in_memory_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
569        ) {
570            let test_dir = temp_dir_create!();
571            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
572                .with_immutables(&[1, 2])
573                .with_in_memory_ledger_snapshots(&[737])
574                .build();
575            let snapshotter =
576                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
577            let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
578            fs::create_dir(&ancillary_snapshot_dir).unwrap();
579
580            snapshotter
581                .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
582                .await
583                .unwrap();
584
585            assert_dir_eq!(
586                &ancillary_snapshot_dir,
587                format!(
588                    "* {IMMUTABLE_DIR}/
589                     ** 00002.chunk
590                     ** 00002.primary
591                     ** 00002.secondary
592                     * {LEDGER_DIR}/
593                     ** 737/
594                     *** {}/
595                     **** {}
596                     *** {}
597                     *** {}",
598                    LedgerStateSnapshot::IN_MEMORY_TABLES,
599                    LedgerStateSnapshot::IN_MEMORY_TVAR,
600                    LedgerStateSnapshot::IN_MEMORY_META,
601                    LedgerStateSnapshot::IN_MEMORY_STATE,
602                )
603            );
604        }
605
606        #[tokio::test]
607        async fn getting_files_to_include_fails_when_no_ledger_file_found() {
608            let test_dir = temp_dir_create!();
609            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
610                .with_immutables(&[1, 2])
611                .build();
612            let snapshotter =
613                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
614            let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
615            fs::create_dir(&ancillary_snapshot_dir).unwrap();
616
617            snapshotter
618                .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
619                .await
620                .expect_err("Should fail if no ledger file found");
621        }
622
623        #[tokio::test]
624        async fn delete_temporary_working_directory_after_snapshot_is_created() {
625            let test_dir = temp_dir_create!();
626            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
627                .with_immutables(&[1, 2])
628                .with_legacy_ledger_snapshots(&[637])
629                .build();
630            let snapshotter = CompressedArchiveSnapshotter {
631                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
632                    fake_keys::signable_manifest_signature()[0],
633                )),
634                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
635            };
636
637            snapshotter
638                .snapshot_ancillary(1, "ancillary")
639                .await
640                .unwrap();
641
642            let temp_ancillary_snapshot_dir =
643                snapshotter.temp_ancillary_snapshot_directory("ancillary");
644            assert!(
645                !temp_ancillary_snapshot_dir.exists(),
646                "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
647                temp_ancillary_snapshot_dir.display()
648            );
649        }
650
651        #[tokio::test]
652        async fn delete_temporary_working_directory_even_if_snapshot_creation_fails() {
653            let test_dir = temp_dir_create!();
654            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
655                .with_immutables(&[1, 2])
656                .with_legacy_ledger_snapshots(&[637])
657                .build();
658            let snapshotter = CompressedArchiveSnapshotter {
659                ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message("failure")),
660                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
661            };
662
663            snapshotter
664                .snapshot_ancillary(1, "ancillary")
665                .await
666                .unwrap_err();
667
668            let temp_ancillary_snapshot_dir =
669                snapshotter.temp_ancillary_snapshot_directory("ancillary");
670            assert!(
671                !temp_ancillary_snapshot_dir.exists(),
672                "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
673                temp_ancillary_snapshot_dir.display()
674            );
675        }
676
677        #[tokio::test]
678        async fn create_archive_should_embed_only_two_last_ledgers_and_last_immutables() {
679            let test_dir = temp_dir_create!();
680            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
681                .with_immutables(&[1, 2, 3])
682                .with_legacy_ledger_snapshots(&[437, 537, 637, 737])
683                .with_non_ledger_files(&["9not_included"])
684                .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
685                .build();
686            fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
687
688            let snapshotter = CompressedArchiveSnapshotter {
689                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
690                    fake_keys::signable_manifest_signature()[0],
691                )),
692                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
693            };
694
695            let snapshot = snapshotter
696                .snapshot_ancillary(2, "ancillary")
697                .await
698                .unwrap();
699
700            let unpack_dir = snapshot.unpack_gzip(&test_dir);
701            assert_dir_eq!(
702                &unpack_dir,
703                // Only the two last ledger files should be included
704                format!(
705                    "* {IMMUTABLE_DIR}/
706                     ** 00003.chunk
707                     ** 00003.primary
708                     ** 00003.secondary
709                     * {LEDGER_DIR}/
710                     ** 637
711                     ** 737
712                     * {}",
713                    AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME
714                )
715            );
716        }
717
718        #[tokio::test]
719        async fn create_archive_fail_if_manifest_signing_fail() {
720            let test_dir = temp_dir_create!();
721            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
722                .with_immutables(&[1, 2])
723                .with_legacy_ledger_snapshots(&[737])
724                .build();
725
726            let snapshotter = CompressedArchiveSnapshotter {
727                ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message(
728                    "MockAncillarySigner failed",
729                )),
730                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
731            };
732
733            let err = snapshotter
734                .snapshot_ancillary(1, "ancillary")
735                .await
736                .expect_err("Must fail if manifest signing fails");
737            assert!(
738                err.to_string().contains("MockAncillarySigner failed"),
739                "Expected error message to be raised by the mock ancillary signer, but got: '{err:?}'",
740            );
741        }
742
743        #[tokio::test]
744        async fn create_archive_of_legacy_ledger_snapshot_generate_sign_and_include_manifest_file()
745        {
746            let test_dir = temp_dir_create!();
747            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
748                .with_immutables(&[1, 2, 3])
749                .with_legacy_ledger_snapshots(&[537, 637, 737])
750                .with_non_immutables(&["not_to_include.txt"])
751                .build();
752            File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
753
754            let snapshotter = CompressedArchiveSnapshotter {
755                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
756                    fake_keys::signable_manifest_signature()[0],
757                )),
758                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
759            };
760
761            let archive = snapshotter
762                .snapshot_ancillary(2, "ancillary")
763                .await
764                .unwrap();
765            let unpacked = archive.unpack_gzip(test_dir);
766            let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
767
768            assert!(manifest_path.exists());
769
770            let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
771                File::open(&manifest_path).unwrap(),
772            )
773            .unwrap();
774
775            assert_eq!(
776                vec![
777                    PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
778                    PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
779                    PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
780                    PathBuf::from(LEDGER_DIR).join("637"),
781                    PathBuf::from(LEDGER_DIR).join("737"),
782                ],
783                manifest.files()
784            );
785            assert_eq!(
786                Some(
787                    fake_keys::signable_manifest_signature()[0]
788                        .try_into()
789                        .unwrap()
790                ),
791                manifest.signature()
792            )
793        }
794
795        #[tokio::test]
796        async fn create_archive_of_in_memory_ledger_snapshot_generate_sign_and_include_manifest_file(
797        ) {
798            let test_dir = temp_dir_create!();
799            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
800                .with_immutables(&[1, 2, 3])
801                .with_in_memory_ledger_snapshots(&[537, 637, 737])
802                .with_non_immutables(&["not_to_include.txt"])
803                .build();
804            File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
805
806            let snapshotter = CompressedArchiveSnapshotter {
807                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
808                    fake_keys::signable_manifest_signature()[0],
809                )),
810                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
811            };
812
813            let archive = snapshotter
814                .snapshot_ancillary(2, "ancillary")
815                .await
816                .unwrap();
817            let unpacked = archive.unpack_gzip(test_dir);
818            let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
819
820            assert!(manifest_path.exists());
821
822            let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
823                File::open(&manifest_path).unwrap(),
824            )
825            .unwrap();
826
827            assert_eq!(
828                vec![
829                    PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
830                    PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
831                    PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
832                    PathBuf::from(LEDGER_DIR)
833                        .join("637")
834                        .join(LedgerStateSnapshot::IN_MEMORY_META),
835                    PathBuf::from(LEDGER_DIR)
836                        .join("637")
837                        .join(LedgerStateSnapshot::IN_MEMORY_STATE),
838                    PathBuf::from(LEDGER_DIR)
839                        .join("637")
840                        .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
841                        .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
842                    PathBuf::from(LEDGER_DIR)
843                        .join("737")
844                        .join(LedgerStateSnapshot::IN_MEMORY_META),
845                    PathBuf::from(LEDGER_DIR)
846                        .join("737")
847                        .join(LedgerStateSnapshot::IN_MEMORY_STATE),
848                    PathBuf::from(LEDGER_DIR)
849                        .join("737")
850                        .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
851                        .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
852                ],
853                manifest.files()
854            );
855            assert_eq!(
856                Some(
857                    fake_keys::signable_manifest_signature()[0]
858                        .try_into()
859                        .unwrap()
860                ),
861                manifest.signature()
862            )
863        }
864    }
865
866    mod compute_immutable_total_and_average_uncompressed_size {
867        use mithril_common::current_function;
868
869        use super::*;
870
871        #[tokio::test]
872        async fn should_compute_the_total_size_of_the_immutables() {
873            let test_dir = temp_dir_create!();
874            let immutable_trio_file_size = 777;
875
876            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
877                .with_immutables(&[1, 2, 3])
878                .set_immutable_trio_file_size(immutable_trio_file_size)
879                .with_legacy_ledger_snapshots(&[737])
880                .set_ledger_file_size(6666)
881                .with_volatile_files(&["blocks-0.dat"])
882                .set_volatile_file_size(99)
883                .build();
884
885            let snapshotter =
886                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
887
888            let sizes = snapshotter
889                .compute_immutable_files_total_uncompressed_size(2)
890                .await
891                .unwrap();
892
893            assert_eq!(immutable_trio_file_size * 2, sizes)
894        }
895
896        #[tokio::test]
897        async fn should_return_an_error_when_compute_up_to_immutable_0() {
898            let test_dir = temp_dir_create!();
899            let cardano_db = DummyCardanoDbBuilder::new(current_function!()).build();
900
901            let snapshotter =
902                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
903
904            snapshotter
905                .compute_immutable_files_total_uncompressed_size(0)
906                .await
907                .expect_err("Should return an error when no immutable file number");
908        }
909    }
910}