mithril_aggregator/services/snapshotter/
compressed_archive_snapshotter.rs

1use anyhow::{Context, anyhow};
2use async_trait::async_trait;
3use slog::{Logger, debug, warn};
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use mithril_cardano_node_internal_database::entities::AncillaryFilesManifest;
9use mithril_cardano_node_internal_database::entities::{ImmutableFile, LedgerStateSnapshot};
10use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, immutable_trio_names};
11use mithril_common::StdResult;
12use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber};
13use mithril_common::logging::LoggerExtensions;
14
15use crate::dependency_injection::DependenciesBuilderError;
16use crate::tools::file_archiver::appender::{AppenderData, AppenderEntries, TarAppender};
17use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver};
18use crate::tools::file_size;
19
20use super::{Snapshotter, ancillary_signer::AncillarySigner};
21
22/// Compressed Archive Snapshotter create a compressed file.
23pub struct CompressedArchiveSnapshotter {
24    /// DB directory to snapshot
25    db_directory: PathBuf,
26
27    /// Directory to store ongoing snapshot
28    ongoing_snapshot_directory: PathBuf,
29
30    /// Compression algorithm to use
31    compression_algorithm: CompressionAlgorithm,
32
33    file_archiver: Arc<FileArchiver>,
34
35    ancillary_signer: Arc<dyn AncillarySigner>,
36
37    logger: Logger,
38}
39
40#[async_trait]
41impl Snapshotter for CompressedArchiveSnapshotter {
42    async fn snapshot_all_completed_immutables(
43        &self,
44        archive_name_without_extension: &str,
45    ) -> StdResult<FileArchive> {
46        debug!(
47            self.logger,
48            "Snapshotting all completed immutables into archive: '{archive_name_without_extension}'"
49        );
50
51        let paths_to_include = ImmutableFile::list_completed_in_dir(&self.db_directory)
52            .with_context(|| {
53                format!(
54                    "Can not list completed immutables in database directory: '{}'",
55                    self.db_directory.display()
56                )
57            })?
58            .into_iter()
59            .map(|immutable_file: ImmutableFile| {
60                PathBuf::from(IMMUTABLE_DIR).join(immutable_file.filename)
61            })
62            .collect();
63        let appender = AppenderEntries::new(paths_to_include, self.db_directory.clone())?;
64        self.snapshot(archive_name_without_extension, appender).await
65    }
66
67    async fn snapshot_ancillary(
68        &self,
69        immutable_file_number: ImmutableFileNumber,
70        archive_name_without_extension: &str,
71    ) -> StdResult<FileArchive> {
72        debug!(
73            self.logger,
74            "Snapshotting ancillary archive: '{archive_name_without_extension}'"
75        );
76
77        let temp_snapshot_directory =
78            self.temp_ancillary_snapshot_directory(archive_name_without_extension);
79        fs::create_dir(&temp_snapshot_directory).with_context(|| {
80            format!(
81                "Can not create temporary snapshot ancillary directory: '{}'",
82                temp_snapshot_directory.display()
83            )
84        })?;
85
86        let snapshot_result = self
87            .snapshot_ancillary_from_temp_directory(
88                immutable_file_number,
89                &temp_snapshot_directory,
90                archive_name_without_extension,
91            )
92            .await;
93
94        if let Err(e) = fs::remove_dir_all(&temp_snapshot_directory) {
95            warn!(
96                self.logger, "Failed to remove temporary snapshot ancillary directory '{}'", temp_snapshot_directory.display();
97                "error" => ?e
98            );
99        }
100        snapshot_result
101    }
102
103    async fn snapshot_immutable_trio(
104        &self,
105        immutable_file_number: ImmutableFileNumber,
106        archive_name_without_extension: &str,
107    ) -> StdResult<FileArchive> {
108        debug!(
109            self.logger,
110            "Snapshotting immutable trio {immutable_file_number} into archive '{archive_name_without_extension}'"
111        );
112
113        let files_to_archive = immutable_trio_names(immutable_file_number)
114            .iter()
115            .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
116            .collect();
117        let appender = AppenderEntries::new(files_to_archive, self.db_directory.clone())?;
118
119        self.snapshot(archive_name_without_extension, appender).await
120    }
121
122    async fn compute_immutable_files_total_uncompressed_size(
123        &self,
124        up_to_immutable_file_number: ImmutableFileNumber,
125    ) -> StdResult<u64> {
126        if up_to_immutable_file_number == 0 {
127            return Err(anyhow!(
128                "Could not compute the total size without immutable files"
129            ));
130        }
131        let immutable_directory = self.db_directory.join(IMMUTABLE_DIR);
132
133        tokio::task::spawn_blocking(move || -> StdResult<_> {
134            let immutable_paths = (1..=up_to_immutable_file_number)
135                .flat_map(immutable_trio_names)
136                .map(|filename| immutable_directory.join(filename))
137                .collect();
138
139            file_size::compute_size(immutable_paths)
140        })
141        .await?
142    }
143
144    fn compression_algorithm(&self) -> CompressionAlgorithm {
145        self.compression_algorithm
146    }
147}
148
149impl CompressedArchiveSnapshotter {
150    /// Snapshotter factory
151    pub fn new(
152        db_directory: PathBuf,
153        ongoing_snapshot_directory: PathBuf,
154        compression_algorithm: CompressionAlgorithm,
155        file_archiver: Arc<FileArchiver>,
156        ancillary_signer: Arc<dyn AncillarySigner>,
157        logger: Logger,
158    ) -> StdResult<CompressedArchiveSnapshotter> {
159        if ongoing_snapshot_directory.exists() {
160            fs::remove_dir_all(&ongoing_snapshot_directory).with_context(|| {
161                format!(
162                    "Can not remove snapshotter directory: '{}'.",
163                    ongoing_snapshot_directory.display()
164                )
165            })?;
166        }
167
168        fs::create_dir(&ongoing_snapshot_directory).map_err(|e| {
169            DependenciesBuilderError::Initialization {
170                message: format!(
171                    "Can not create snapshotter directory: '{}'.",
172                    ongoing_snapshot_directory.display()
173                ),
174                error: Some(e.into()),
175            }
176        })?;
177
178        Ok(Self {
179            db_directory,
180            ongoing_snapshot_directory,
181            compression_algorithm,
182            file_archiver,
183            ancillary_signer,
184            logger: logger.new_with_component_name::<Self>(),
185        })
186    }
187
188    async fn snapshot<T: TarAppender + 'static>(
189        &self,
190        name_without_extension: &str,
191        appender: T,
192    ) -> StdResult<FileArchive> {
193        let file_archiver = self.file_archiver.clone();
194        let parameters = ArchiveParameters {
195            archive_name_without_extension: name_without_extension.to_string(),
196            target_directory: self.ongoing_snapshot_directory.clone(),
197            compression_algorithm: self.compression_algorithm,
198        };
199
200        // spawn a separate thread to prevent blocking
201        let file_archive = tokio::task::spawn_blocking(move || -> StdResult<FileArchive> {
202            file_archiver.archive(parameters, appender)
203        })
204        .await??;
205
206        Ok(file_archive)
207    }
208
209    fn temp_ancillary_snapshot_directory(&self, discriminator: &str) -> PathBuf {
210        self.ongoing_snapshot_directory
211            .join(format!("temp-ancillary-{discriminator}"))
212    }
213
214    async fn snapshot_ancillary_from_temp_directory(
215        &self,
216        immutable_file_number: ImmutableFileNumber,
217        temp_snapshot_directory: &Path,
218        archive_name_without_extension: &str,
219    ) -> StdResult<FileArchive> {
220        let paths_to_include = self
221            .get_files_and_directories_for_ancillary_snapshot(
222                immutable_file_number,
223                temp_snapshot_directory,
224            )
225            .await?;
226        let signed_manifest = self
227            .build_and_sign_ancillary_manifest(paths_to_include.clone(), temp_snapshot_directory)
228            .await?;
229        let appender =
230            AppenderEntries::new(paths_to_include, temp_snapshot_directory.to_path_buf())?.chain(
231                AppenderData::from_json(
232                    PathBuf::from(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME),
233                    &signed_manifest,
234                )?,
235            );
236        self.snapshot(archive_name_without_extension, appender).await
237    }
238
239    /// Returns the list of files and directories to include in ancillary snapshot.
240    ///
241    /// Those files will be copied to the target folder in order to have fixed copies that do not
242    /// change during the snapshot process.
243    ///
244    /// The immutable file included in the ancillary archive corresponds to the last one (and not finalized yet)
245    /// when the immutable file number given to the function corresponds to the penultimate.
246    async fn get_files_and_directories_for_ancillary_snapshot(
247        &self,
248        immutable_file_number: u64,
249        target_folder: &Path,
250    ) -> StdResult<Vec<PathBuf>> {
251        let next_immutable_file_number = immutable_file_number + 1;
252        let mut files_to_snapshot: Vec<PathBuf> = immutable_trio_names(next_immutable_file_number)
253            .into_iter()
254            .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
255            .collect();
256
257        let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
258        let ledger_files = LedgerStateSnapshot::list_all_in_dir(&db_ledger_dir)?;
259        let latest_ledger_files: Vec<PathBuf> = ledger_files
260            .iter()
261            .rev()
262            .take(2)
263            .flat_map(|ledger_state_snapshot| ledger_state_snapshot.get_files_relative_path())
264            .map(|path| PathBuf::from(LEDGER_DIR).join(path))
265            .collect();
266        if latest_ledger_files.is_empty() {
267            return Err(anyhow!(
268                "No ledger state snapshot found in the ledger directory: '{}'",
269                db_ledger_dir.display()
270            ));
271        }
272        files_to_snapshot.extend(latest_ledger_files);
273
274        fs::create_dir(target_folder.join(IMMUTABLE_DIR))
275            .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
276        fs::create_dir(target_folder.join(LEDGER_DIR))
277            .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
278
279        for file in &files_to_snapshot {
280            // Some files to snapshot are in subfolders (i.e.: in-memory ledger snapshots files)
281            if let Some(parent_dir) = file.parent() {
282                let target_parent_dir = target_folder.join(parent_dir);
283                if !target_parent_dir.exists() {
284                    fs::create_dir_all(&target_parent_dir).with_context(|| {
285                        format!("Can not create folder: `{}`", target_parent_dir.display())
286                    })?;
287                }
288            }
289
290            let source = self.db_directory.join(file);
291            let target = target_folder.join(file);
292            tokio::fs::copy(&source, &target).await.with_context(|| {
293                format!(
294                    "Failed to copy file `{}` to `{}`",
295                    source.display(),
296                    target.display()
297                )
298            })?;
299        }
300
301        Ok(files_to_snapshot)
302    }
303
304    async fn build_and_sign_ancillary_manifest(
305        &self,
306        paths_to_include: Vec<PathBuf>,
307        temp_snapshot_directory: &Path,
308    ) -> StdResult<AncillaryFilesManifest> {
309        let mut manifest =
310            AncillaryFilesManifest::from_paths(temp_snapshot_directory, paths_to_include).await?;
311        let signature = self
312            .ancillary_signer
313            .compute_ancillary_manifest_signature(&manifest)
314            .await?;
315        manifest.set_signature(signature);
316
317        Ok(manifest)
318    }
319}
320
321#[cfg(test)]
322mod tests {
323    use std::fs::File;
324    use std::path::Path;
325    use std::sync::Arc;
326
327    use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder;
328    use mithril_common::test_utils::assert_equivalent;
329    use mithril_common::{assert_dir_eq, current_function, temp_dir_create};
330
331    use crate::services::ancillary_signer::MockAncillarySigner;
332    use crate::test_tools::TestLogger;
333
334    use super::*;
335
336    fn list_files(test_dir: &Path) -> Vec<String> {
337        fs::read_dir(test_dir)
338            .unwrap()
339            .map(|f| f.unwrap().file_name().to_str().unwrap().to_owned())
340            .collect()
341    }
342
343    fn snapshotter_for_test(
344        test_directory: &Path,
345        db_directory: &Path,
346        compression_algorithm: CompressionAlgorithm,
347    ) -> CompressedArchiveSnapshotter {
348        CompressedArchiveSnapshotter::new(
349            db_directory.to_path_buf(),
350            test_directory.join("ongoing_snapshot"),
351            compression_algorithm,
352            Arc::new(FileArchiver::new_for_test(
353                test_directory.join("verification"),
354            )),
355            Arc::new(MockAncillarySigner::new()),
356            TestLogger::stdout(),
357        )
358        .unwrap()
359    }
360
361    #[test]
362    fn return_parametrized_compression_algorithm() {
363        let test_dir = temp_dir_create!();
364        let snapshotter = snapshotter_for_test(
365            &test_dir,
366            Path::new("whatever"),
367            CompressionAlgorithm::Zstandard,
368        );
369
370        assert_eq!(
371            CompressionAlgorithm::Zstandard,
372            snapshotter.compression_algorithm()
373        );
374    }
375
376    #[test]
377    fn should_create_directory_if_does_not_exist() {
378        let test_dir = temp_dir_create!();
379        let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
380        let db_directory = test_dir.join("whatever");
381
382        CompressedArchiveSnapshotter::new(
383            db_directory,
384            ongoing_snapshot_directory.clone(),
385            CompressionAlgorithm::Gzip,
386            Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
387            Arc::new(MockAncillarySigner::new()),
388            TestLogger::stdout(),
389        )
390        .unwrap();
391
392        assert!(ongoing_snapshot_directory.is_dir());
393    }
394
395    #[test]
396    fn should_clean_ongoing_snapshot_directory_if_already_exists() {
397        let test_dir = temp_dir_create!();
398        let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
399        let db_directory = test_dir.join("whatever");
400
401        fs::create_dir_all(&ongoing_snapshot_directory).unwrap();
402
403        File::create(ongoing_snapshot_directory.join("whatever.txt")).unwrap();
404
405        CompressedArchiveSnapshotter::new(
406            db_directory,
407            ongoing_snapshot_directory.clone(),
408            CompressionAlgorithm::Gzip,
409            Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
410            Arc::new(MockAncillarySigner::new()),
411            TestLogger::stdout(),
412        )
413        .unwrap();
414
415        assert_eq!(0, fs::read_dir(ongoing_snapshot_directory).unwrap().count());
416    }
417
418    #[tokio::test]
419    async fn should_create_snapshots_in_its_ongoing_snapshot_directory() {
420        let test_dir = temp_dir_create!();
421        let pending_snapshot_directory = test_dir.join("pending_snapshot");
422        let cardano_db = DummyCardanoDbBuilder::new(current_function!())
423            .with_immutables(&[1])
424            .append_immutable_trio()
425            .build();
426
427        let snapshotter = CompressedArchiveSnapshotter::new(
428            cardano_db.get_dir().clone(),
429            pending_snapshot_directory.clone(),
430            CompressionAlgorithm::Gzip,
431            Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
432            Arc::new(MockAncillarySigner::new()),
433            TestLogger::stdout(),
434        )
435        .unwrap();
436        let snapshot = snapshotter
437            .snapshot_all_completed_immutables("whatever")
438            .await
439            .unwrap();
440
441        assert_eq!(
442            pending_snapshot_directory,
443            snapshot.get_file_path().parent().unwrap()
444        );
445    }
446
447    mod snapshot_all_completed_immutables {
448        use super::*;
449
450        #[tokio::test]
451        async fn include_only_completed_immutables() {
452            let test_dir = temp_dir_create!();
453            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
454                .with_immutables(&[1, 2, 3])
455                .append_immutable_trio()
456                .with_legacy_ledger_snapshots(&[437])
457                .with_volatile_files(&["blocks-0.dat"])
458                .with_non_immutables(&["random_file.txt", "00002.trap"])
459                .build();
460
461            let snapshotter =
462                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
463
464            let snapshot = snapshotter
465                .snapshot_all_completed_immutables("completed_immutables")
466                .await
467                .unwrap();
468
469            let unpack_dir = snapshot.unpack_gzip(&test_dir);
470            let unpacked_files = list_files(&unpack_dir);
471            let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
472
473            assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
474            assert_equivalent(
475                vec![
476                    "00001.chunk".to_string(),
477                    "00001.primary".to_string(),
478                    "00001.secondary".to_string(),
479                    "00002.chunk".to_string(),
480                    "00002.primary".to_string(),
481                    "00002.secondary".to_string(),
482                    "00003.chunk".to_string(),
483                    "00003.primary".to_string(),
484                    "00003.secondary".to_string(),
485                ],
486                unpacked_immutable_files,
487            );
488        }
489    }
490
491    mod snapshot_immutable_trio {
492        use super::*;
493
494        #[tokio::test]
495        async fn include_only_immutable_trio() {
496            let test_dir = temp_dir_create!();
497            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
498                .with_immutables(&[1, 2, 3])
499                .with_legacy_ledger_snapshots(&[437])
500                .with_volatile_files(&["blocks-0.dat"])
501                .with_non_immutables(&["random_file.txt", "00002.trap"])
502                .build();
503
504            let snapshotter =
505                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
506
507            let snapshot = snapshotter.snapshot_immutable_trio(2, "immutable-2").await.unwrap();
508
509            let unpack_dir = snapshot.unpack_gzip(&test_dir);
510            let unpacked_files = list_files(&unpack_dir);
511            let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
512
513            assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
514            assert_equivalent(
515                vec![
516                    "00002.chunk".to_string(),
517                    "00002.primary".to_string(),
518                    "00002.secondary".to_string(),
519                ],
520                unpacked_immutable_files,
521            );
522        }
523    }
524
525    mod snapshot_ancillary {
526        use mithril_common::test_utils::fake_keys;
527
528        use super::*;
529
530        #[tokio::test]
531        async fn getting_files_to_include_for_legacy_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure()
532         {
533            let test_dir = temp_dir_create!();
534            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
535                .with_immutables(&[1, 2])
536                .with_legacy_ledger_snapshots(&[737])
537                .build();
538            let snapshotter =
539                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
540            let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
541            fs::create_dir(&ancillary_snapshot_dir).unwrap();
542
543            snapshotter
544                .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
545                .await
546                .unwrap();
547
548            assert_dir_eq!(
549                &ancillary_snapshot_dir,
550                format!(
551                    "* {IMMUTABLE_DIR}/
552                     ** 00002.chunk
553                     ** 00002.primary
554                     ** 00002.secondary
555                     * {LEDGER_DIR}/
556                     ** 737"
557                )
558            );
559        }
560
561        #[tokio::test]
562        async fn getting_files_to_include_for_in_memory_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure()
563         {
564            let test_dir = temp_dir_create!();
565            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
566                .with_immutables(&[1, 2])
567                .with_in_memory_ledger_snapshots(&[737])
568                .build();
569            let snapshotter =
570                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
571            let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
572            fs::create_dir(&ancillary_snapshot_dir).unwrap();
573
574            snapshotter
575                .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
576                .await
577                .unwrap();
578
579            assert_dir_eq!(
580                &ancillary_snapshot_dir,
581                format!(
582                    "* {IMMUTABLE_DIR}/
583                     ** 00002.chunk
584                     ** 00002.primary
585                     ** 00002.secondary
586                     * {LEDGER_DIR}/
587                     ** 737/
588                     *** {}/
589                     **** {}
590                     *** {}
591                     *** {}",
592                    LedgerStateSnapshot::IN_MEMORY_TABLES,
593                    LedgerStateSnapshot::IN_MEMORY_TVAR,
594                    LedgerStateSnapshot::IN_MEMORY_META,
595                    LedgerStateSnapshot::IN_MEMORY_STATE,
596                )
597            );
598        }
599
600        #[tokio::test]
601        async fn getting_files_to_include_fails_when_no_ledger_file_found() {
602            let test_dir = temp_dir_create!();
603            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
604                .with_immutables(&[1, 2])
605                .build();
606            let snapshotter =
607                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
608            let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
609            fs::create_dir(&ancillary_snapshot_dir).unwrap();
610
611            snapshotter
612                .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
613                .await
614                .expect_err("Should fail if no ledger file found");
615        }
616
617        #[tokio::test]
618        async fn delete_temporary_working_directory_after_snapshot_is_created() {
619            let test_dir = temp_dir_create!();
620            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
621                .with_immutables(&[1, 2])
622                .with_legacy_ledger_snapshots(&[637])
623                .build();
624            let snapshotter = CompressedArchiveSnapshotter {
625                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
626                    fake_keys::signable_manifest_signature()[0],
627                )),
628                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
629            };
630
631            snapshotter.snapshot_ancillary(1, "ancillary").await.unwrap();
632
633            let temp_ancillary_snapshot_dir =
634                snapshotter.temp_ancillary_snapshot_directory("ancillary");
635            assert!(
636                !temp_ancillary_snapshot_dir.exists(),
637                "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
638                temp_ancillary_snapshot_dir.display()
639            );
640        }
641
642        #[tokio::test]
643        async fn delete_temporary_working_directory_even_if_snapshot_creation_fails() {
644            let test_dir = temp_dir_create!();
645            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
646                .with_immutables(&[1, 2])
647                .with_legacy_ledger_snapshots(&[637])
648                .build();
649            let snapshotter = CompressedArchiveSnapshotter {
650                ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message("failure")),
651                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
652            };
653
654            snapshotter.snapshot_ancillary(1, "ancillary").await.unwrap_err();
655
656            let temp_ancillary_snapshot_dir =
657                snapshotter.temp_ancillary_snapshot_directory("ancillary");
658            assert!(
659                !temp_ancillary_snapshot_dir.exists(),
660                "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
661                temp_ancillary_snapshot_dir.display()
662            );
663        }
664
665        #[tokio::test]
666        async fn create_archive_should_embed_only_two_last_ledgers_and_last_immutables() {
667            let test_dir = temp_dir_create!();
668            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
669                .with_immutables(&[1, 2, 3])
670                .with_legacy_ledger_snapshots(&[437, 537, 637, 737])
671                .with_non_ledger_files(&["9not_included"])
672                .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
673                .build();
674            fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
675
676            let snapshotter = CompressedArchiveSnapshotter {
677                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
678                    fake_keys::signable_manifest_signature()[0],
679                )),
680                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
681            };
682
683            let snapshot = snapshotter.snapshot_ancillary(2, "ancillary").await.unwrap();
684
685            let unpack_dir = snapshot.unpack_gzip(&test_dir);
686            assert_dir_eq!(
687                &unpack_dir,
688                // Only the two last ledger files should be included
689                format!(
690                    "* {IMMUTABLE_DIR}/
691                     ** 00003.chunk
692                     ** 00003.primary
693                     ** 00003.secondary
694                     * {LEDGER_DIR}/
695                     ** 637
696                     ** 737
697                     * {}",
698                    AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME
699                )
700            );
701        }
702
703        #[tokio::test]
704        async fn create_archive_fail_if_manifest_signing_fail() {
705            let test_dir = temp_dir_create!();
706            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
707                .with_immutables(&[1, 2])
708                .with_legacy_ledger_snapshots(&[737])
709                .build();
710
711            let snapshotter = CompressedArchiveSnapshotter {
712                ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message(
713                    "MockAncillarySigner failed",
714                )),
715                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
716            };
717
718            let err = snapshotter
719                .snapshot_ancillary(1, "ancillary")
720                .await
721                .expect_err("Must fail if manifest signing fails");
722            assert!(
723                err.to_string().contains("MockAncillarySigner failed"),
724                "Expected error message to be raised by the mock ancillary signer, but got: '{err:?}'",
725            );
726        }
727
728        #[tokio::test]
729        async fn create_archive_of_legacy_ledger_snapshot_generate_sign_and_include_manifest_file()
730        {
731            let test_dir = temp_dir_create!();
732            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
733                .with_immutables(&[1, 2, 3])
734                .with_legacy_ledger_snapshots(&[537, 637, 737])
735                .with_non_immutables(&["not_to_include.txt"])
736                .build();
737            File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
738
739            let snapshotter = CompressedArchiveSnapshotter {
740                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
741                    fake_keys::signable_manifest_signature()[0],
742                )),
743                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
744            };
745
746            let archive = snapshotter.snapshot_ancillary(2, "ancillary").await.unwrap();
747            let unpacked = archive.unpack_gzip(test_dir);
748            let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
749
750            assert!(manifest_path.exists());
751
752            let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
753                File::open(&manifest_path).unwrap(),
754            )
755            .unwrap();
756
757            assert_eq!(
758                vec![
759                    PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
760                    PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
761                    PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
762                    PathBuf::from(LEDGER_DIR).join("637"),
763                    PathBuf::from(LEDGER_DIR).join("737"),
764                ],
765                manifest.files()
766            );
767            assert_eq!(
768                Some(fake_keys::signable_manifest_signature()[0].try_into().unwrap()),
769                manifest.signature()
770            )
771        }
772
773        #[tokio::test]
774        async fn create_archive_of_in_memory_ledger_snapshot_generate_sign_and_include_manifest_file()
775         {
776            let test_dir = temp_dir_create!();
777            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
778                .with_immutables(&[1, 2, 3])
779                .with_in_memory_ledger_snapshots(&[537, 637, 737])
780                .with_non_immutables(&["not_to_include.txt"])
781                .build();
782            File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
783
784            let snapshotter = CompressedArchiveSnapshotter {
785                ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
786                    fake_keys::signable_manifest_signature()[0],
787                )),
788                ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
789            };
790
791            let archive = snapshotter.snapshot_ancillary(2, "ancillary").await.unwrap();
792            let unpacked = archive.unpack_gzip(test_dir);
793            let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
794
795            assert!(manifest_path.exists());
796
797            let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
798                File::open(&manifest_path).unwrap(),
799            )
800            .unwrap();
801
802            assert_eq!(
803                vec![
804                    PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
805                    PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
806                    PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
807                    PathBuf::from(LEDGER_DIR)
808                        .join("637")
809                        .join(LedgerStateSnapshot::IN_MEMORY_META),
810                    PathBuf::from(LEDGER_DIR)
811                        .join("637")
812                        .join(LedgerStateSnapshot::IN_MEMORY_STATE),
813                    PathBuf::from(LEDGER_DIR)
814                        .join("637")
815                        .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
816                        .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
817                    PathBuf::from(LEDGER_DIR)
818                        .join("737")
819                        .join(LedgerStateSnapshot::IN_MEMORY_META),
820                    PathBuf::from(LEDGER_DIR)
821                        .join("737")
822                        .join(LedgerStateSnapshot::IN_MEMORY_STATE),
823                    PathBuf::from(LEDGER_DIR)
824                        .join("737")
825                        .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
826                        .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
827                ],
828                manifest.files()
829            );
830            assert_eq!(
831                Some(fake_keys::signable_manifest_signature()[0].try_into().unwrap()),
832                manifest.signature()
833            )
834        }
835    }
836
837    mod compute_immutable_total_and_average_uncompressed_size {
838        use mithril_common::current_function;
839
840        use super::*;
841
842        #[tokio::test]
843        async fn should_compute_the_total_size_of_the_immutables() {
844            let test_dir = temp_dir_create!();
845            let immutable_trio_file_size = 777;
846
847            let cardano_db = DummyCardanoDbBuilder::new(current_function!())
848                .with_immutables(&[1, 2, 3])
849                .set_immutable_trio_file_size(immutable_trio_file_size)
850                .with_legacy_ledger_snapshots(&[737])
851                .set_ledger_file_size(6666)
852                .with_volatile_files(&["blocks-0.dat"])
853                .set_volatile_file_size(99)
854                .build();
855
856            let snapshotter =
857                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
858
859            let sizes = snapshotter
860                .compute_immutable_files_total_uncompressed_size(2)
861                .await
862                .unwrap();
863
864            assert_eq!(immutable_trio_file_size * 2, sizes)
865        }
866
867        #[tokio::test]
868        async fn should_return_an_error_when_compute_up_to_immutable_0() {
869            let test_dir = temp_dir_create!();
870            let cardano_db = DummyCardanoDbBuilder::new(current_function!()).build();
871
872            let snapshotter =
873                snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
874
875            snapshotter
876                .compute_immutable_files_total_uncompressed_size(0)
877                .await
878                .expect_err("Should return an error when no immutable file number");
879        }
880    }
881}