1use anyhow::{anyhow, Context};
2use async_trait::async_trait;
3use slog::{debug, warn, Logger};
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use mithril_common::digesters::{
9 immutable_trio_names, ImmutableFile, LedgerStateSnapshot, IMMUTABLE_DIR, LEDGER_DIR,
10};
11use mithril_common::entities::{AncillaryFilesManifest, CompressionAlgorithm, ImmutableFileNumber};
12use mithril_common::logging::LoggerExtensions;
13use mithril_common::StdResult;
14
15use crate::dependency_injection::DependenciesBuilderError;
16use crate::tools::file_archiver::appender::{AppenderData, AppenderEntries, TarAppender};
17use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver};
18use crate::tools::file_size;
19
20use super::{ancillary_signer::AncillarySigner, Snapshotter};
21
22pub struct CompressedArchiveSnapshotter {
24 db_directory: PathBuf,
26
27 ongoing_snapshot_directory: PathBuf,
29
30 compression_algorithm: CompressionAlgorithm,
32
33 file_archiver: Arc<FileArchiver>,
34
35 ancillary_signer: Arc<dyn AncillarySigner>,
36
37 logger: Logger,
38}
39
40#[async_trait]
41impl Snapshotter for CompressedArchiveSnapshotter {
42 async fn snapshot_all_completed_immutables(
43 &self,
44 archive_name_without_extension: &str,
45 ) -> StdResult<FileArchive> {
46 debug!(
47 self.logger,
48 "Snapshotting all completed immutables into archive: '{archive_name_without_extension}'"
49 );
50
51 let paths_to_include = ImmutableFile::list_completed_in_dir(&self.db_directory)
52 .with_context(|| {
53 format!(
54 "Can not list completed immutables in database directory: '{}'",
55 self.db_directory.display()
56 )
57 })?
58 .into_iter()
59 .map(|immutable_file: ImmutableFile| {
60 PathBuf::from(IMMUTABLE_DIR).join(immutable_file.filename)
61 })
62 .collect();
63 let appender = AppenderEntries::new(paths_to_include, self.db_directory.clone())?;
64 self.snapshot(archive_name_without_extension, appender)
65 .await
66 }
67
68 async fn snapshot_ancillary(
69 &self,
70 immutable_file_number: ImmutableFileNumber,
71 archive_name_without_extension: &str,
72 ) -> StdResult<FileArchive> {
73 debug!(
74 self.logger,
75 "Snapshotting ancillary archive: '{archive_name_without_extension}'"
76 );
77
78 let temp_snapshot_directory =
79 self.temp_ancillary_snapshot_directory(archive_name_without_extension);
80 fs::create_dir(&temp_snapshot_directory).with_context(|| {
81 format!(
82 "Can not create temporary snapshot ancillary directory: '{}'",
83 temp_snapshot_directory.display()
84 )
85 })?;
86
87 let snapshot_result = self
88 .snapshot_ancillary_from_temp_directory(
89 immutable_file_number,
90 &temp_snapshot_directory,
91 archive_name_without_extension,
92 )
93 .await;
94
95 if let Err(e) = fs::remove_dir_all(&temp_snapshot_directory) {
96 warn!(
97 self.logger, "Failed to remove temporary snapshot ancillary directory '{}'", temp_snapshot_directory.display();
98 "error" => ?e
99 );
100 }
101 snapshot_result
102 }
103
104 async fn snapshot_immutable_trio(
105 &self,
106 immutable_file_number: ImmutableFileNumber,
107 archive_name_without_extension: &str,
108 ) -> StdResult<FileArchive> {
109 debug!(
110 self.logger,
111 "Snapshotting immutable trio {immutable_file_number} into archive '{archive_name_without_extension}'"
112 );
113
114 let files_to_archive = immutable_trio_names(immutable_file_number)
115 .iter()
116 .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
117 .collect();
118 let appender = AppenderEntries::new(files_to_archive, self.db_directory.clone())?;
119
120 self.snapshot(archive_name_without_extension, appender)
121 .await
122 }
123
124 async fn compute_immutable_files_total_uncompressed_size(
125 &self,
126 up_to_immutable_file_number: ImmutableFileNumber,
127 ) -> StdResult<u64> {
128 if up_to_immutable_file_number == 0 {
129 return Err(anyhow!(
130 "Could not compute the total size without immutable files"
131 ));
132 }
133 let immutable_directory = self.db_directory.join(IMMUTABLE_DIR);
134
135 tokio::task::spawn_blocking(move || -> StdResult<_> {
136 let immutable_paths = (1..=up_to_immutable_file_number)
137 .flat_map(immutable_trio_names)
138 .map(|filename| immutable_directory.join(filename))
139 .collect();
140
141 file_size::compute_size(immutable_paths)
142 })
143 .await?
144 }
145
146 fn compression_algorithm(&self) -> CompressionAlgorithm {
147 self.compression_algorithm
148 }
149}
150
151impl CompressedArchiveSnapshotter {
152 pub fn new(
154 db_directory: PathBuf,
155 ongoing_snapshot_directory: PathBuf,
156 compression_algorithm: CompressionAlgorithm,
157 file_archiver: Arc<FileArchiver>,
158 ancillary_signer: Arc<dyn AncillarySigner>,
159 logger: Logger,
160 ) -> StdResult<CompressedArchiveSnapshotter> {
161 if ongoing_snapshot_directory.exists() {
162 fs::remove_dir_all(&ongoing_snapshot_directory).with_context(|| {
163 format!(
164 "Can not remove snapshotter directory: '{}'.",
165 ongoing_snapshot_directory.display()
166 )
167 })?;
168 }
169
170 fs::create_dir(&ongoing_snapshot_directory).map_err(|e| {
171 DependenciesBuilderError::Initialization {
172 message: format!(
173 "Can not create snapshotter directory: '{}'.",
174 ongoing_snapshot_directory.display()
175 ),
176 error: Some(e.into()),
177 }
178 })?;
179
180 Ok(Self {
181 db_directory,
182 ongoing_snapshot_directory,
183 compression_algorithm,
184 file_archiver,
185 ancillary_signer,
186 logger: logger.new_with_component_name::<Self>(),
187 })
188 }
189
190 async fn snapshot<T: TarAppender + 'static>(
191 &self,
192 name_without_extension: &str,
193 appender: T,
194 ) -> StdResult<FileArchive> {
195 let file_archiver = self.file_archiver.clone();
196 let parameters = ArchiveParameters {
197 archive_name_without_extension: name_without_extension.to_string(),
198 target_directory: self.ongoing_snapshot_directory.clone(),
199 compression_algorithm: self.compression_algorithm,
200 };
201
202 let file_archive = tokio::task::spawn_blocking(move || -> StdResult<FileArchive> {
204 file_archiver.archive(parameters, appender)
205 })
206 .await??;
207
208 Ok(file_archive)
209 }
210
211 fn temp_ancillary_snapshot_directory(&self, discriminator: &str) -> PathBuf {
212 self.ongoing_snapshot_directory
213 .join(format!("temp-ancillary-{discriminator}"))
214 }
215
216 async fn snapshot_ancillary_from_temp_directory(
217 &self,
218 immutable_file_number: ImmutableFileNumber,
219 temp_snapshot_directory: &Path,
220 archive_name_without_extension: &str,
221 ) -> StdResult<FileArchive> {
222 let paths_to_include = self
223 .get_files_and_directories_for_ancillary_snapshot(
224 immutable_file_number,
225 temp_snapshot_directory,
226 )
227 .await?;
228 let signed_manifest = self
229 .build_and_sign_ancillary_manifest(paths_to_include.clone(), temp_snapshot_directory)
230 .await?;
231 let appender =
232 AppenderEntries::new(paths_to_include, temp_snapshot_directory.to_path_buf())?.chain(
233 AppenderData::from_json(
234 PathBuf::from(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME),
235 &signed_manifest,
236 )?,
237 );
238 self.snapshot(archive_name_without_extension, appender)
239 .await
240 }
241
242 async fn get_files_and_directories_for_ancillary_snapshot(
250 &self,
251 immutable_file_number: u64,
252 target_folder: &Path,
253 ) -> StdResult<Vec<PathBuf>> {
254 let next_immutable_file_number = immutable_file_number + 1;
255 let mut files_to_snapshot: Vec<PathBuf> = immutable_trio_names(next_immutable_file_number)
256 .into_iter()
257 .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
258 .collect();
259
260 let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
261 let ledger_files = LedgerStateSnapshot::list_all_in_dir(&db_ledger_dir)?;
262 let latest_ledger_files: Vec<PathBuf> = ledger_files
263 .iter()
264 .rev()
265 .take(2)
266 .flat_map(|ledger_state_snapshot| ledger_state_snapshot.get_files_relative_path())
267 .map(|path| PathBuf::from(LEDGER_DIR).join(path))
268 .collect();
269 if latest_ledger_files.is_empty() {
270 return Err(anyhow!(
271 "No ledger state snapshot found in the ledger directory: '{}'",
272 db_ledger_dir.display()
273 ));
274 }
275 files_to_snapshot.extend(latest_ledger_files);
276
277 fs::create_dir(target_folder.join(IMMUTABLE_DIR))
278 .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
279 fs::create_dir(target_folder.join(LEDGER_DIR))
280 .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
281
282 for file in &files_to_snapshot {
283 if let Some(parent_dir) = file.parent() {
285 let target_parent_dir = target_folder.join(parent_dir);
286 if !target_parent_dir.exists() {
287 fs::create_dir_all(&target_parent_dir).with_context(|| {
288 format!("Can not create folder: `{}`", target_parent_dir.display())
289 })?;
290 }
291 }
292
293 let source = self.db_directory.join(file);
294 let target = target_folder.join(file);
295 tokio::fs::copy(&source, &target).await.with_context(|| {
296 format!(
297 "Failed to copy file `{}` to `{}`",
298 source.display(),
299 target.display()
300 )
301 })?;
302 }
303
304 Ok(files_to_snapshot)
305 }
306
307 async fn build_and_sign_ancillary_manifest(
308 &self,
309 paths_to_include: Vec<PathBuf>,
310 temp_snapshot_directory: &Path,
311 ) -> StdResult<AncillaryFilesManifest> {
312 let manifest =
313 AncillaryFilesManifest::from_paths(temp_snapshot_directory, paths_to_include).await?;
314 let signature = self
315 .ancillary_signer
316 .compute_ancillary_manifest_signature(&manifest)
317 .await?;
318 let signed_manifest = AncillaryFilesManifest {
319 signature: Some(signature),
320 ..manifest
321 };
322
323 Ok(signed_manifest)
324 }
325}
326
327#[cfg(test)]
328mod tests {
329 use std::fs::File;
330 use std::path::Path;
331 use std::sync::Arc;
332
333 use mithril_common::digesters::DummyCardanoDbBuilder;
334 use mithril_common::test_utils::assert_equivalent;
335 use mithril_common::{assert_dir_eq, current_function, temp_dir_create};
336
337 use crate::services::ancillary_signer::MockAncillarySigner;
338 use crate::test_tools::TestLogger;
339
340 use super::*;
341
342 fn list_files(test_dir: &Path) -> Vec<String> {
343 fs::read_dir(test_dir)
344 .unwrap()
345 .map(|f| f.unwrap().file_name().to_str().unwrap().to_owned())
346 .collect()
347 }
348
349 fn snapshotter_for_test(
350 test_directory: &Path,
351 db_directory: &Path,
352 compression_algorithm: CompressionAlgorithm,
353 ) -> CompressedArchiveSnapshotter {
354 CompressedArchiveSnapshotter::new(
355 db_directory.to_path_buf(),
356 test_directory.join("ongoing_snapshot"),
357 compression_algorithm,
358 Arc::new(FileArchiver::new_for_test(
359 test_directory.join("verification"),
360 )),
361 Arc::new(MockAncillarySigner::new()),
362 TestLogger::stdout(),
363 )
364 .unwrap()
365 }
366
367 #[test]
368 fn return_parametrized_compression_algorithm() {
369 let test_dir = temp_dir_create!();
370 let snapshotter = snapshotter_for_test(
371 &test_dir,
372 Path::new("whatever"),
373 CompressionAlgorithm::Zstandard,
374 );
375
376 assert_eq!(
377 CompressionAlgorithm::Zstandard,
378 snapshotter.compression_algorithm()
379 );
380 }
381
382 #[test]
383 fn should_create_directory_if_does_not_exist() {
384 let test_dir = temp_dir_create!();
385 let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
386 let db_directory = test_dir.join("whatever");
387
388 CompressedArchiveSnapshotter::new(
389 db_directory,
390 ongoing_snapshot_directory.clone(),
391 CompressionAlgorithm::Gzip,
392 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
393 Arc::new(MockAncillarySigner::new()),
394 TestLogger::stdout(),
395 )
396 .unwrap();
397
398 assert!(ongoing_snapshot_directory.is_dir());
399 }
400
401 #[test]
402 fn should_clean_ongoing_snapshot_directory_if_already_exists() {
403 let test_dir = temp_dir_create!();
404 let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
405 let db_directory = test_dir.join("whatever");
406
407 fs::create_dir_all(&ongoing_snapshot_directory).unwrap();
408
409 File::create(ongoing_snapshot_directory.join("whatever.txt")).unwrap();
410
411 CompressedArchiveSnapshotter::new(
412 db_directory,
413 ongoing_snapshot_directory.clone(),
414 CompressionAlgorithm::Gzip,
415 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
416 Arc::new(MockAncillarySigner::new()),
417 TestLogger::stdout(),
418 )
419 .unwrap();
420
421 assert_eq!(0, fs::read_dir(ongoing_snapshot_directory).unwrap().count());
422 }
423
424 #[tokio::test]
425 async fn should_create_snapshots_in_its_ongoing_snapshot_directory() {
426 let test_dir = temp_dir_create!();
427 let pending_snapshot_directory = test_dir.join("pending_snapshot");
428 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
429 .with_immutables(&[1])
430 .append_immutable_trio()
431 .build();
432
433 let snapshotter = CompressedArchiveSnapshotter::new(
434 cardano_db.get_dir().clone(),
435 pending_snapshot_directory.clone(),
436 CompressionAlgorithm::Gzip,
437 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
438 Arc::new(MockAncillarySigner::new()),
439 TestLogger::stdout(),
440 )
441 .unwrap();
442 let snapshot = snapshotter
443 .snapshot_all_completed_immutables("whatever")
444 .await
445 .unwrap();
446
447 assert_eq!(
448 pending_snapshot_directory,
449 snapshot.get_file_path().parent().unwrap()
450 );
451 }
452
453 mod snapshot_all_completed_immutables {
454 use super::*;
455
456 #[tokio::test]
457 async fn include_only_completed_immutables() {
458 let test_dir = temp_dir_create!();
459 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
460 .with_immutables(&[1, 2, 3])
461 .append_immutable_trio()
462 .with_legacy_ledger_snapshots(&[437])
463 .with_volatile_files(&["blocks-0.dat"])
464 .with_non_immutables(&["random_file.txt", "00002.trap"])
465 .build();
466
467 let snapshotter =
468 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
469
470 let snapshot = snapshotter
471 .snapshot_all_completed_immutables("completed_immutables")
472 .await
473 .unwrap();
474
475 let unpack_dir = snapshot.unpack_gzip(&test_dir);
476 let unpacked_files = list_files(&unpack_dir);
477 let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
478
479 assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
480 assert_equivalent(
481 vec![
482 "00001.chunk".to_string(),
483 "00001.primary".to_string(),
484 "00001.secondary".to_string(),
485 "00002.chunk".to_string(),
486 "00002.primary".to_string(),
487 "00002.secondary".to_string(),
488 "00003.chunk".to_string(),
489 "00003.primary".to_string(),
490 "00003.secondary".to_string(),
491 ],
492 unpacked_immutable_files,
493 );
494 }
495 }
496
497 mod snapshot_immutable_trio {
498 use super::*;
499
500 #[tokio::test]
501 async fn include_only_immutable_trio() {
502 let test_dir = temp_dir_create!();
503 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
504 .with_immutables(&[1, 2, 3])
505 .with_legacy_ledger_snapshots(&[437])
506 .with_volatile_files(&["blocks-0.dat"])
507 .with_non_immutables(&["random_file.txt", "00002.trap"])
508 .build();
509
510 let snapshotter =
511 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
512
513 let snapshot = snapshotter
514 .snapshot_immutable_trio(2, "immutable-2")
515 .await
516 .unwrap();
517
518 let unpack_dir = snapshot.unpack_gzip(&test_dir);
519 let unpacked_files = list_files(&unpack_dir);
520 let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
521
522 assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
523 assert_equivalent(
524 vec![
525 "00002.chunk".to_string(),
526 "00002.primary".to_string(),
527 "00002.secondary".to_string(),
528 ],
529 unpacked_immutable_files,
530 );
531 }
532 }
533
534 mod snapshot_ancillary {
535 use mithril_common::test_utils::fake_keys;
536
537 use super::*;
538
539 #[tokio::test]
540 async fn getting_files_to_include_for_legacy_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
541 ) {
542 let test_dir = temp_dir_create!();
543 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
544 .with_immutables(&[1, 2])
545 .with_legacy_ledger_snapshots(&[737])
546 .build();
547 let snapshotter =
548 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
549 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
550 fs::create_dir(&ancillary_snapshot_dir).unwrap();
551
552 snapshotter
553 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
554 .await
555 .unwrap();
556
557 assert_dir_eq!(
558 &ancillary_snapshot_dir,
559 format!(
560 "* {IMMUTABLE_DIR}/
561 ** 00002.chunk
562 ** 00002.primary
563 ** 00002.secondary
564 * {LEDGER_DIR}/
565 ** 737"
566 )
567 );
568 }
569
570 #[tokio::test]
571 async fn getting_files_to_include_for_in_memory_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
572 ) {
573 let test_dir = temp_dir_create!();
574 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
575 .with_immutables(&[1, 2])
576 .with_in_memory_ledger_snapshots(&[737])
577 .build();
578 let snapshotter =
579 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
580 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
581 fs::create_dir(&ancillary_snapshot_dir).unwrap();
582
583 snapshotter
584 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
585 .await
586 .unwrap();
587
588 assert_dir_eq!(
589 &ancillary_snapshot_dir,
590 format!(
591 "* {IMMUTABLE_DIR}/
592 ** 00002.chunk
593 ** 00002.primary
594 ** 00002.secondary
595 * {LEDGER_DIR}/
596 ** 737/
597 *** {}/
598 **** {}
599 *** {}
600 *** {}",
601 LedgerStateSnapshot::IN_MEMORY_TABLES,
602 LedgerStateSnapshot::IN_MEMORY_TVAR,
603 LedgerStateSnapshot::IN_MEMORY_META,
604 LedgerStateSnapshot::IN_MEMORY_STATE,
605 )
606 );
607 }
608
609 #[tokio::test]
610 async fn getting_files_to_include_fails_when_no_ledger_file_found() {
611 let test_dir = temp_dir_create!();
612 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
613 .with_immutables(&[1, 2])
614 .build();
615 let snapshotter =
616 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
617 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
618 fs::create_dir(&ancillary_snapshot_dir).unwrap();
619
620 snapshotter
621 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
622 .await
623 .expect_err("Should fail if no ledger file found");
624 }
625
626 #[tokio::test]
627 async fn delete_temporary_working_directory_after_snapshot_is_created() {
628 let test_dir = temp_dir_create!();
629 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
630 .with_immutables(&[1, 2])
631 .with_legacy_ledger_snapshots(&[637])
632 .build();
633 let snapshotter = CompressedArchiveSnapshotter {
634 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
635 fake_keys::signable_manifest_signature()[0],
636 )),
637 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
638 };
639
640 snapshotter
641 .snapshot_ancillary(1, "ancillary")
642 .await
643 .unwrap();
644
645 let temp_ancillary_snapshot_dir =
646 snapshotter.temp_ancillary_snapshot_directory("ancillary");
647 assert!(
648 !temp_ancillary_snapshot_dir.exists(),
649 "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
650 temp_ancillary_snapshot_dir.display()
651 );
652 }
653
654 #[tokio::test]
655 async fn delete_temporary_working_directory_even_if_snapshot_creation_fails() {
656 let test_dir = temp_dir_create!();
657 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
658 .with_immutables(&[1, 2])
659 .with_legacy_ledger_snapshots(&[637])
660 .build();
661 let snapshotter = CompressedArchiveSnapshotter {
662 ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message("failure")),
663 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
664 };
665
666 snapshotter
667 .snapshot_ancillary(1, "ancillary")
668 .await
669 .unwrap_err();
670
671 let temp_ancillary_snapshot_dir =
672 snapshotter.temp_ancillary_snapshot_directory("ancillary");
673 assert!(
674 !temp_ancillary_snapshot_dir.exists(),
675 "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
676 temp_ancillary_snapshot_dir.display()
677 );
678 }
679
680 #[tokio::test]
681 async fn create_archive_should_embed_only_two_last_ledgers_and_last_immutables() {
682 let test_dir = temp_dir_create!();
683 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
684 .with_immutables(&[1, 2, 3])
685 .with_legacy_ledger_snapshots(&[437, 537, 637, 737])
686 .with_non_ledger_files(&["9not_included"])
687 .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
688 .build();
689 fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
690
691 let snapshotter = CompressedArchiveSnapshotter {
692 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
693 fake_keys::signable_manifest_signature()[0],
694 )),
695 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
696 };
697
698 let snapshot = snapshotter
699 .snapshot_ancillary(2, "ancillary")
700 .await
701 .unwrap();
702
703 let unpack_dir = snapshot.unpack_gzip(&test_dir);
704 assert_dir_eq!(
705 &unpack_dir,
706 format!(
708 "* {IMMUTABLE_DIR}/
709 ** 00003.chunk
710 ** 00003.primary
711 ** 00003.secondary
712 * {LEDGER_DIR}/
713 ** 637
714 ** 737
715 * {}",
716 AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME
717 )
718 );
719 }
720
721 #[tokio::test]
722 async fn create_archive_fail_if_manifest_signing_fail() {
723 let test_dir = temp_dir_create!();
724 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
725 .with_immutables(&[1, 2])
726 .with_legacy_ledger_snapshots(&[737])
727 .build();
728
729 let snapshotter = CompressedArchiveSnapshotter {
730 ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message(
731 "MockAncillarySigner failed",
732 )),
733 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
734 };
735
736 let err = snapshotter
737 .snapshot_ancillary(1, "ancillary")
738 .await
739 .expect_err("Must fail if manifest signing fails");
740 assert!(
741 err.to_string().contains("MockAncillarySigner failed"),
742 "Expected error message to be raised by the mock ancillary signer, but got: '{err:?}'",
743 );
744 }
745
746 #[tokio::test]
747 async fn create_archive_of_legacy_ledger_snapshot_generate_sign_and_include_manifest_file()
748 {
749 let test_dir = temp_dir_create!();
750 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
751 .with_immutables(&[1, 2, 3])
752 .with_legacy_ledger_snapshots(&[537, 637, 737])
753 .with_non_immutables(&["not_to_include.txt"])
754 .build();
755 File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
756
757 let snapshotter = CompressedArchiveSnapshotter {
758 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
759 fake_keys::signable_manifest_signature()[0],
760 )),
761 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
762 };
763
764 let archive = snapshotter
765 .snapshot_ancillary(2, "ancillary")
766 .await
767 .unwrap();
768 let unpacked = archive.unpack_gzip(test_dir);
769 let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
770
771 assert!(manifest_path.exists());
772
773 let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
774 File::open(&manifest_path).unwrap(),
775 )
776 .unwrap();
777
778 assert_eq!(
779 vec![
780 &PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
781 &PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
782 &PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
783 &PathBuf::from(LEDGER_DIR).join("637"),
784 &PathBuf::from(LEDGER_DIR).join("737"),
785 ],
786 manifest.data.keys().collect::<Vec<_>>()
787 );
788 assert_eq!(
789 Some(
790 fake_keys::signable_manifest_signature()[0]
791 .try_into()
792 .unwrap()
793 ),
794 manifest.signature
795 )
796 }
797
798 #[tokio::test]
799 async fn create_archive_of_in_memory_ledger_snapshot_generate_sign_and_include_manifest_file(
800 ) {
801 let test_dir = temp_dir_create!();
802 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
803 .with_immutables(&[1, 2, 3])
804 .with_in_memory_ledger_snapshots(&[537, 637, 737])
805 .with_non_immutables(&["not_to_include.txt"])
806 .build();
807 File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
808
809 let snapshotter = CompressedArchiveSnapshotter {
810 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
811 fake_keys::signable_manifest_signature()[0],
812 )),
813 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
814 };
815
816 let archive = snapshotter
817 .snapshot_ancillary(2, "ancillary")
818 .await
819 .unwrap();
820 let unpacked = archive.unpack_gzip(test_dir);
821 let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
822
823 assert!(manifest_path.exists());
824
825 let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
826 File::open(&manifest_path).unwrap(),
827 )
828 .unwrap();
829
830 assert_eq!(
831 vec![
832 &PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
833 &PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
834 &PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
835 &PathBuf::from(LEDGER_DIR)
836 .join("637")
837 .join(LedgerStateSnapshot::IN_MEMORY_META),
838 &PathBuf::from(LEDGER_DIR)
839 .join("637")
840 .join(LedgerStateSnapshot::IN_MEMORY_STATE),
841 &PathBuf::from(LEDGER_DIR)
842 .join("637")
843 .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
844 .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
845 &PathBuf::from(LEDGER_DIR)
846 .join("737")
847 .join(LedgerStateSnapshot::IN_MEMORY_META),
848 &PathBuf::from(LEDGER_DIR)
849 .join("737")
850 .join(LedgerStateSnapshot::IN_MEMORY_STATE),
851 &PathBuf::from(LEDGER_DIR)
852 .join("737")
853 .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
854 .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
855 ],
856 manifest.data.keys().collect::<Vec<_>>()
857 );
858 assert_eq!(
859 Some(
860 fake_keys::signable_manifest_signature()[0]
861 .try_into()
862 .unwrap()
863 ),
864 manifest.signature
865 )
866 }
867 }
868
869 mod compute_immutable_total_and_average_uncompressed_size {
870 use mithril_common::current_function;
871
872 use super::*;
873
874 #[tokio::test]
875 async fn should_compute_the_total_size_of_the_immutables() {
876 let test_dir = temp_dir_create!();
877 let immutable_trio_file_size = 777;
878
879 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
880 .with_immutables(&[1, 2, 3])
881 .set_immutable_trio_file_size(immutable_trio_file_size)
882 .with_legacy_ledger_snapshots(&[737])
883 .set_ledger_file_size(6666)
884 .with_volatile_files(&["blocks-0.dat"])
885 .set_volatile_file_size(99)
886 .build();
887
888 let snapshotter =
889 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
890
891 let sizes = snapshotter
892 .compute_immutable_files_total_uncompressed_size(2)
893 .await
894 .unwrap();
895
896 assert_eq!(immutable_trio_file_size * 2, sizes)
897 }
898
899 #[tokio::test]
900 async fn should_return_an_error_when_compute_up_to_immutable_0() {
901 let test_dir = temp_dir_create!();
902 let cardano_db = DummyCardanoDbBuilder::new(current_function!()).build();
903
904 let snapshotter =
905 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
906
907 snapshotter
908 .compute_immutable_files_total_uncompressed_size(0)
909 .await
910 .expect_err("Should return an error when no immutable file number");
911 }
912 }
913}