1use anyhow::{anyhow, Context};
2use async_trait::async_trait;
3use slog::{debug, warn, Logger};
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use mithril_cardano_node_internal_database::entities::AncillaryFilesManifest;
9use mithril_cardano_node_internal_database::entities::{ImmutableFile, LedgerStateSnapshot};
10use mithril_cardano_node_internal_database::{immutable_trio_names, IMMUTABLE_DIR, LEDGER_DIR};
11use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber};
12use mithril_common::logging::LoggerExtensions;
13use mithril_common::StdResult;
14
15use crate::dependency_injection::DependenciesBuilderError;
16use crate::tools::file_archiver::appender::{AppenderData, AppenderEntries, TarAppender};
17use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver};
18use crate::tools::file_size;
19
20use super::{ancillary_signer::AncillarySigner, Snapshotter};
21
22pub struct CompressedArchiveSnapshotter {
24 db_directory: PathBuf,
26
27 ongoing_snapshot_directory: PathBuf,
29
30 compression_algorithm: CompressionAlgorithm,
32
33 file_archiver: Arc<FileArchiver>,
34
35 ancillary_signer: Arc<dyn AncillarySigner>,
36
37 logger: Logger,
38}
39
40#[async_trait]
41impl Snapshotter for CompressedArchiveSnapshotter {
42 async fn snapshot_all_completed_immutables(
43 &self,
44 archive_name_without_extension: &str,
45 ) -> StdResult<FileArchive> {
46 debug!(
47 self.logger,
48 "Snapshotting all completed immutables into archive: '{archive_name_without_extension}'"
49 );
50
51 let paths_to_include = ImmutableFile::list_completed_in_dir(&self.db_directory)
52 .with_context(|| {
53 format!(
54 "Can not list completed immutables in database directory: '{}'",
55 self.db_directory.display()
56 )
57 })?
58 .into_iter()
59 .map(|immutable_file: ImmutableFile| {
60 PathBuf::from(IMMUTABLE_DIR).join(immutable_file.filename)
61 })
62 .collect();
63 let appender = AppenderEntries::new(paths_to_include, self.db_directory.clone())?;
64 self.snapshot(archive_name_without_extension, appender)
65 .await
66 }
67
68 async fn snapshot_ancillary(
69 &self,
70 immutable_file_number: ImmutableFileNumber,
71 archive_name_without_extension: &str,
72 ) -> StdResult<FileArchive> {
73 debug!(
74 self.logger,
75 "Snapshotting ancillary archive: '{archive_name_without_extension}'"
76 );
77
78 let temp_snapshot_directory =
79 self.temp_ancillary_snapshot_directory(archive_name_without_extension);
80 fs::create_dir(&temp_snapshot_directory).with_context(|| {
81 format!(
82 "Can not create temporary snapshot ancillary directory: '{}'",
83 temp_snapshot_directory.display()
84 )
85 })?;
86
87 let snapshot_result = self
88 .snapshot_ancillary_from_temp_directory(
89 immutable_file_number,
90 &temp_snapshot_directory,
91 archive_name_without_extension,
92 )
93 .await;
94
95 if let Err(e) = fs::remove_dir_all(&temp_snapshot_directory) {
96 warn!(
97 self.logger, "Failed to remove temporary snapshot ancillary directory '{}'", temp_snapshot_directory.display();
98 "error" => ?e
99 );
100 }
101 snapshot_result
102 }
103
104 async fn snapshot_immutable_trio(
105 &self,
106 immutable_file_number: ImmutableFileNumber,
107 archive_name_without_extension: &str,
108 ) -> StdResult<FileArchive> {
109 debug!(
110 self.logger,
111 "Snapshotting immutable trio {immutable_file_number} into archive '{archive_name_without_extension}'"
112 );
113
114 let files_to_archive = immutable_trio_names(immutable_file_number)
115 .iter()
116 .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
117 .collect();
118 let appender = AppenderEntries::new(files_to_archive, self.db_directory.clone())?;
119
120 self.snapshot(archive_name_without_extension, appender)
121 .await
122 }
123
124 async fn compute_immutable_files_total_uncompressed_size(
125 &self,
126 up_to_immutable_file_number: ImmutableFileNumber,
127 ) -> StdResult<u64> {
128 if up_to_immutable_file_number == 0 {
129 return Err(anyhow!(
130 "Could not compute the total size without immutable files"
131 ));
132 }
133 let immutable_directory = self.db_directory.join(IMMUTABLE_DIR);
134
135 tokio::task::spawn_blocking(move || -> StdResult<_> {
136 let immutable_paths = (1..=up_to_immutable_file_number)
137 .flat_map(immutable_trio_names)
138 .map(|filename| immutable_directory.join(filename))
139 .collect();
140
141 file_size::compute_size(immutable_paths)
142 })
143 .await?
144 }
145
146 fn compression_algorithm(&self) -> CompressionAlgorithm {
147 self.compression_algorithm
148 }
149}
150
151impl CompressedArchiveSnapshotter {
152 pub fn new(
154 db_directory: PathBuf,
155 ongoing_snapshot_directory: PathBuf,
156 compression_algorithm: CompressionAlgorithm,
157 file_archiver: Arc<FileArchiver>,
158 ancillary_signer: Arc<dyn AncillarySigner>,
159 logger: Logger,
160 ) -> StdResult<CompressedArchiveSnapshotter> {
161 if ongoing_snapshot_directory.exists() {
162 fs::remove_dir_all(&ongoing_snapshot_directory).with_context(|| {
163 format!(
164 "Can not remove snapshotter directory: '{}'.",
165 ongoing_snapshot_directory.display()
166 )
167 })?;
168 }
169
170 fs::create_dir(&ongoing_snapshot_directory).map_err(|e| {
171 DependenciesBuilderError::Initialization {
172 message: format!(
173 "Can not create snapshotter directory: '{}'.",
174 ongoing_snapshot_directory.display()
175 ),
176 error: Some(e.into()),
177 }
178 })?;
179
180 Ok(Self {
181 db_directory,
182 ongoing_snapshot_directory,
183 compression_algorithm,
184 file_archiver,
185 ancillary_signer,
186 logger: logger.new_with_component_name::<Self>(),
187 })
188 }
189
190 async fn snapshot<T: TarAppender + 'static>(
191 &self,
192 name_without_extension: &str,
193 appender: T,
194 ) -> StdResult<FileArchive> {
195 let file_archiver = self.file_archiver.clone();
196 let parameters = ArchiveParameters {
197 archive_name_without_extension: name_without_extension.to_string(),
198 target_directory: self.ongoing_snapshot_directory.clone(),
199 compression_algorithm: self.compression_algorithm,
200 };
201
202 let file_archive = tokio::task::spawn_blocking(move || -> StdResult<FileArchive> {
204 file_archiver.archive(parameters, appender)
205 })
206 .await??;
207
208 Ok(file_archive)
209 }
210
211 fn temp_ancillary_snapshot_directory(&self, discriminator: &str) -> PathBuf {
212 self.ongoing_snapshot_directory
213 .join(format!("temp-ancillary-{discriminator}"))
214 }
215
216 async fn snapshot_ancillary_from_temp_directory(
217 &self,
218 immutable_file_number: ImmutableFileNumber,
219 temp_snapshot_directory: &Path,
220 archive_name_without_extension: &str,
221 ) -> StdResult<FileArchive> {
222 let paths_to_include = self
223 .get_files_and_directories_for_ancillary_snapshot(
224 immutable_file_number,
225 temp_snapshot_directory,
226 )
227 .await?;
228 let signed_manifest = self
229 .build_and_sign_ancillary_manifest(paths_to_include.clone(), temp_snapshot_directory)
230 .await?;
231 let appender =
232 AppenderEntries::new(paths_to_include, temp_snapshot_directory.to_path_buf())?.chain(
233 AppenderData::from_json(
234 PathBuf::from(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME),
235 &signed_manifest,
236 )?,
237 );
238 self.snapshot(archive_name_without_extension, appender)
239 .await
240 }
241
242 async fn get_files_and_directories_for_ancillary_snapshot(
250 &self,
251 immutable_file_number: u64,
252 target_folder: &Path,
253 ) -> StdResult<Vec<PathBuf>> {
254 let next_immutable_file_number = immutable_file_number + 1;
255 let mut files_to_snapshot: Vec<PathBuf> = immutable_trio_names(next_immutable_file_number)
256 .into_iter()
257 .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
258 .collect();
259
260 let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
261 let ledger_files = LedgerStateSnapshot::list_all_in_dir(&db_ledger_dir)?;
262 let latest_ledger_files: Vec<PathBuf> = ledger_files
263 .iter()
264 .rev()
265 .take(2)
266 .flat_map(|ledger_state_snapshot| ledger_state_snapshot.get_files_relative_path())
267 .map(|path| PathBuf::from(LEDGER_DIR).join(path))
268 .collect();
269 if latest_ledger_files.is_empty() {
270 return Err(anyhow!(
271 "No ledger state snapshot found in the ledger directory: '{}'",
272 db_ledger_dir.display()
273 ));
274 }
275 files_to_snapshot.extend(latest_ledger_files);
276
277 fs::create_dir(target_folder.join(IMMUTABLE_DIR))
278 .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
279 fs::create_dir(target_folder.join(LEDGER_DIR))
280 .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
281
282 for file in &files_to_snapshot {
283 if let Some(parent_dir) = file.parent() {
285 let target_parent_dir = target_folder.join(parent_dir);
286 if !target_parent_dir.exists() {
287 fs::create_dir_all(&target_parent_dir).with_context(|| {
288 format!("Can not create folder: `{}`", target_parent_dir.display())
289 })?;
290 }
291 }
292
293 let source = self.db_directory.join(file);
294 let target = target_folder.join(file);
295 tokio::fs::copy(&source, &target).await.with_context(|| {
296 format!(
297 "Failed to copy file `{}` to `{}`",
298 source.display(),
299 target.display()
300 )
301 })?;
302 }
303
304 Ok(files_to_snapshot)
305 }
306
307 async fn build_and_sign_ancillary_manifest(
308 &self,
309 paths_to_include: Vec<PathBuf>,
310 temp_snapshot_directory: &Path,
311 ) -> StdResult<AncillaryFilesManifest> {
312 let mut manifest =
313 AncillaryFilesManifest::from_paths(temp_snapshot_directory, paths_to_include).await?;
314 let signature = self
315 .ancillary_signer
316 .compute_ancillary_manifest_signature(&manifest)
317 .await?;
318 manifest.set_signature(signature);
319
320 Ok(manifest)
321 }
322}
323
324#[cfg(test)]
325mod tests {
326 use std::fs::File;
327 use std::path::Path;
328 use std::sync::Arc;
329
330 use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder;
331 use mithril_common::test_utils::assert_equivalent;
332 use mithril_common::{assert_dir_eq, current_function, temp_dir_create};
333
334 use crate::services::ancillary_signer::MockAncillarySigner;
335 use crate::test_tools::TestLogger;
336
337 use super::*;
338
339 fn list_files(test_dir: &Path) -> Vec<String> {
340 fs::read_dir(test_dir)
341 .unwrap()
342 .map(|f| f.unwrap().file_name().to_str().unwrap().to_owned())
343 .collect()
344 }
345
346 fn snapshotter_for_test(
347 test_directory: &Path,
348 db_directory: &Path,
349 compression_algorithm: CompressionAlgorithm,
350 ) -> CompressedArchiveSnapshotter {
351 CompressedArchiveSnapshotter::new(
352 db_directory.to_path_buf(),
353 test_directory.join("ongoing_snapshot"),
354 compression_algorithm,
355 Arc::new(FileArchiver::new_for_test(
356 test_directory.join("verification"),
357 )),
358 Arc::new(MockAncillarySigner::new()),
359 TestLogger::stdout(),
360 )
361 .unwrap()
362 }
363
364 #[test]
365 fn return_parametrized_compression_algorithm() {
366 let test_dir = temp_dir_create!();
367 let snapshotter = snapshotter_for_test(
368 &test_dir,
369 Path::new("whatever"),
370 CompressionAlgorithm::Zstandard,
371 );
372
373 assert_eq!(
374 CompressionAlgorithm::Zstandard,
375 snapshotter.compression_algorithm()
376 );
377 }
378
379 #[test]
380 fn should_create_directory_if_does_not_exist() {
381 let test_dir = temp_dir_create!();
382 let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
383 let db_directory = test_dir.join("whatever");
384
385 CompressedArchiveSnapshotter::new(
386 db_directory,
387 ongoing_snapshot_directory.clone(),
388 CompressionAlgorithm::Gzip,
389 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
390 Arc::new(MockAncillarySigner::new()),
391 TestLogger::stdout(),
392 )
393 .unwrap();
394
395 assert!(ongoing_snapshot_directory.is_dir());
396 }
397
398 #[test]
399 fn should_clean_ongoing_snapshot_directory_if_already_exists() {
400 let test_dir = temp_dir_create!();
401 let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
402 let db_directory = test_dir.join("whatever");
403
404 fs::create_dir_all(&ongoing_snapshot_directory).unwrap();
405
406 File::create(ongoing_snapshot_directory.join("whatever.txt")).unwrap();
407
408 CompressedArchiveSnapshotter::new(
409 db_directory,
410 ongoing_snapshot_directory.clone(),
411 CompressionAlgorithm::Gzip,
412 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
413 Arc::new(MockAncillarySigner::new()),
414 TestLogger::stdout(),
415 )
416 .unwrap();
417
418 assert_eq!(0, fs::read_dir(ongoing_snapshot_directory).unwrap().count());
419 }
420
421 #[tokio::test]
422 async fn should_create_snapshots_in_its_ongoing_snapshot_directory() {
423 let test_dir = temp_dir_create!();
424 let pending_snapshot_directory = test_dir.join("pending_snapshot");
425 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
426 .with_immutables(&[1])
427 .append_immutable_trio()
428 .build();
429
430 let snapshotter = CompressedArchiveSnapshotter::new(
431 cardano_db.get_dir().clone(),
432 pending_snapshot_directory.clone(),
433 CompressionAlgorithm::Gzip,
434 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
435 Arc::new(MockAncillarySigner::new()),
436 TestLogger::stdout(),
437 )
438 .unwrap();
439 let snapshot = snapshotter
440 .snapshot_all_completed_immutables("whatever")
441 .await
442 .unwrap();
443
444 assert_eq!(
445 pending_snapshot_directory,
446 snapshot.get_file_path().parent().unwrap()
447 );
448 }
449
450 mod snapshot_all_completed_immutables {
451 use super::*;
452
453 #[tokio::test]
454 async fn include_only_completed_immutables() {
455 let test_dir = temp_dir_create!();
456 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
457 .with_immutables(&[1, 2, 3])
458 .append_immutable_trio()
459 .with_legacy_ledger_snapshots(&[437])
460 .with_volatile_files(&["blocks-0.dat"])
461 .with_non_immutables(&["random_file.txt", "00002.trap"])
462 .build();
463
464 let snapshotter =
465 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
466
467 let snapshot = snapshotter
468 .snapshot_all_completed_immutables("completed_immutables")
469 .await
470 .unwrap();
471
472 let unpack_dir = snapshot.unpack_gzip(&test_dir);
473 let unpacked_files = list_files(&unpack_dir);
474 let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
475
476 assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
477 assert_equivalent(
478 vec![
479 "00001.chunk".to_string(),
480 "00001.primary".to_string(),
481 "00001.secondary".to_string(),
482 "00002.chunk".to_string(),
483 "00002.primary".to_string(),
484 "00002.secondary".to_string(),
485 "00003.chunk".to_string(),
486 "00003.primary".to_string(),
487 "00003.secondary".to_string(),
488 ],
489 unpacked_immutable_files,
490 );
491 }
492 }
493
494 mod snapshot_immutable_trio {
495 use super::*;
496
497 #[tokio::test]
498 async fn include_only_immutable_trio() {
499 let test_dir = temp_dir_create!();
500 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
501 .with_immutables(&[1, 2, 3])
502 .with_legacy_ledger_snapshots(&[437])
503 .with_volatile_files(&["blocks-0.dat"])
504 .with_non_immutables(&["random_file.txt", "00002.trap"])
505 .build();
506
507 let snapshotter =
508 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
509
510 let snapshot = snapshotter
511 .snapshot_immutable_trio(2, "immutable-2")
512 .await
513 .unwrap();
514
515 let unpack_dir = snapshot.unpack_gzip(&test_dir);
516 let unpacked_files = list_files(&unpack_dir);
517 let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
518
519 assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
520 assert_equivalent(
521 vec![
522 "00002.chunk".to_string(),
523 "00002.primary".to_string(),
524 "00002.secondary".to_string(),
525 ],
526 unpacked_immutable_files,
527 );
528 }
529 }
530
531 mod snapshot_ancillary {
532 use mithril_common::test_utils::fake_keys;
533
534 use super::*;
535
536 #[tokio::test]
537 async fn getting_files_to_include_for_legacy_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
538 ) {
539 let test_dir = temp_dir_create!();
540 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
541 .with_immutables(&[1, 2])
542 .with_legacy_ledger_snapshots(&[737])
543 .build();
544 let snapshotter =
545 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
546 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
547 fs::create_dir(&ancillary_snapshot_dir).unwrap();
548
549 snapshotter
550 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
551 .await
552 .unwrap();
553
554 assert_dir_eq!(
555 &ancillary_snapshot_dir,
556 format!(
557 "* {IMMUTABLE_DIR}/
558 ** 00002.chunk
559 ** 00002.primary
560 ** 00002.secondary
561 * {LEDGER_DIR}/
562 ** 737"
563 )
564 );
565 }
566
567 #[tokio::test]
568 async fn getting_files_to_include_for_in_memory_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure(
569 ) {
570 let test_dir = temp_dir_create!();
571 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
572 .with_immutables(&[1, 2])
573 .with_in_memory_ledger_snapshots(&[737])
574 .build();
575 let snapshotter =
576 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
577 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
578 fs::create_dir(&ancillary_snapshot_dir).unwrap();
579
580 snapshotter
581 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
582 .await
583 .unwrap();
584
585 assert_dir_eq!(
586 &ancillary_snapshot_dir,
587 format!(
588 "* {IMMUTABLE_DIR}/
589 ** 00002.chunk
590 ** 00002.primary
591 ** 00002.secondary
592 * {LEDGER_DIR}/
593 ** 737/
594 *** {}/
595 **** {}
596 *** {}
597 *** {}",
598 LedgerStateSnapshot::IN_MEMORY_TABLES,
599 LedgerStateSnapshot::IN_MEMORY_TVAR,
600 LedgerStateSnapshot::IN_MEMORY_META,
601 LedgerStateSnapshot::IN_MEMORY_STATE,
602 )
603 );
604 }
605
606 #[tokio::test]
607 async fn getting_files_to_include_fails_when_no_ledger_file_found() {
608 let test_dir = temp_dir_create!();
609 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
610 .with_immutables(&[1, 2])
611 .build();
612 let snapshotter =
613 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
614 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
615 fs::create_dir(&ancillary_snapshot_dir).unwrap();
616
617 snapshotter
618 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
619 .await
620 .expect_err("Should fail if no ledger file found");
621 }
622
623 #[tokio::test]
624 async fn delete_temporary_working_directory_after_snapshot_is_created() {
625 let test_dir = temp_dir_create!();
626 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
627 .with_immutables(&[1, 2])
628 .with_legacy_ledger_snapshots(&[637])
629 .build();
630 let snapshotter = CompressedArchiveSnapshotter {
631 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
632 fake_keys::signable_manifest_signature()[0],
633 )),
634 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
635 };
636
637 snapshotter
638 .snapshot_ancillary(1, "ancillary")
639 .await
640 .unwrap();
641
642 let temp_ancillary_snapshot_dir =
643 snapshotter.temp_ancillary_snapshot_directory("ancillary");
644 assert!(
645 !temp_ancillary_snapshot_dir.exists(),
646 "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
647 temp_ancillary_snapshot_dir.display()
648 );
649 }
650
651 #[tokio::test]
652 async fn delete_temporary_working_directory_even_if_snapshot_creation_fails() {
653 let test_dir = temp_dir_create!();
654 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
655 .with_immutables(&[1, 2])
656 .with_legacy_ledger_snapshots(&[637])
657 .build();
658 let snapshotter = CompressedArchiveSnapshotter {
659 ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message("failure")),
660 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
661 };
662
663 snapshotter
664 .snapshot_ancillary(1, "ancillary")
665 .await
666 .unwrap_err();
667
668 let temp_ancillary_snapshot_dir =
669 snapshotter.temp_ancillary_snapshot_directory("ancillary");
670 assert!(
671 !temp_ancillary_snapshot_dir.exists(),
672 "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
673 temp_ancillary_snapshot_dir.display()
674 );
675 }
676
677 #[tokio::test]
678 async fn create_archive_should_embed_only_two_last_ledgers_and_last_immutables() {
679 let test_dir = temp_dir_create!();
680 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
681 .with_immutables(&[1, 2, 3])
682 .with_legacy_ledger_snapshots(&[437, 537, 637, 737])
683 .with_non_ledger_files(&["9not_included"])
684 .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
685 .build();
686 fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
687
688 let snapshotter = CompressedArchiveSnapshotter {
689 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
690 fake_keys::signable_manifest_signature()[0],
691 )),
692 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
693 };
694
695 let snapshot = snapshotter
696 .snapshot_ancillary(2, "ancillary")
697 .await
698 .unwrap();
699
700 let unpack_dir = snapshot.unpack_gzip(&test_dir);
701 assert_dir_eq!(
702 &unpack_dir,
703 format!(
705 "* {IMMUTABLE_DIR}/
706 ** 00003.chunk
707 ** 00003.primary
708 ** 00003.secondary
709 * {LEDGER_DIR}/
710 ** 637
711 ** 737
712 * {}",
713 AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME
714 )
715 );
716 }
717
718 #[tokio::test]
719 async fn create_archive_fail_if_manifest_signing_fail() {
720 let test_dir = temp_dir_create!();
721 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
722 .with_immutables(&[1, 2])
723 .with_legacy_ledger_snapshots(&[737])
724 .build();
725
726 let snapshotter = CompressedArchiveSnapshotter {
727 ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message(
728 "MockAncillarySigner failed",
729 )),
730 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
731 };
732
733 let err = snapshotter
734 .snapshot_ancillary(1, "ancillary")
735 .await
736 .expect_err("Must fail if manifest signing fails");
737 assert!(
738 err.to_string().contains("MockAncillarySigner failed"),
739 "Expected error message to be raised by the mock ancillary signer, but got: '{err:?}'",
740 );
741 }
742
743 #[tokio::test]
744 async fn create_archive_of_legacy_ledger_snapshot_generate_sign_and_include_manifest_file()
745 {
746 let test_dir = temp_dir_create!();
747 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
748 .with_immutables(&[1, 2, 3])
749 .with_legacy_ledger_snapshots(&[537, 637, 737])
750 .with_non_immutables(&["not_to_include.txt"])
751 .build();
752 File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
753
754 let snapshotter = CompressedArchiveSnapshotter {
755 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
756 fake_keys::signable_manifest_signature()[0],
757 )),
758 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
759 };
760
761 let archive = snapshotter
762 .snapshot_ancillary(2, "ancillary")
763 .await
764 .unwrap();
765 let unpacked = archive.unpack_gzip(test_dir);
766 let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
767
768 assert!(manifest_path.exists());
769
770 let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
771 File::open(&manifest_path).unwrap(),
772 )
773 .unwrap();
774
775 assert_eq!(
776 vec![
777 PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
778 PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
779 PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
780 PathBuf::from(LEDGER_DIR).join("637"),
781 PathBuf::from(LEDGER_DIR).join("737"),
782 ],
783 manifest.files()
784 );
785 assert_eq!(
786 Some(
787 fake_keys::signable_manifest_signature()[0]
788 .try_into()
789 .unwrap()
790 ),
791 manifest.signature()
792 )
793 }
794
795 #[tokio::test]
796 async fn create_archive_of_in_memory_ledger_snapshot_generate_sign_and_include_manifest_file(
797 ) {
798 let test_dir = temp_dir_create!();
799 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
800 .with_immutables(&[1, 2, 3])
801 .with_in_memory_ledger_snapshots(&[537, 637, 737])
802 .with_non_immutables(&["not_to_include.txt"])
803 .build();
804 File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
805
806 let snapshotter = CompressedArchiveSnapshotter {
807 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
808 fake_keys::signable_manifest_signature()[0],
809 )),
810 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
811 };
812
813 let archive = snapshotter
814 .snapshot_ancillary(2, "ancillary")
815 .await
816 .unwrap();
817 let unpacked = archive.unpack_gzip(test_dir);
818 let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
819
820 assert!(manifest_path.exists());
821
822 let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
823 File::open(&manifest_path).unwrap(),
824 )
825 .unwrap();
826
827 assert_eq!(
828 vec![
829 PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
830 PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
831 PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
832 PathBuf::from(LEDGER_DIR)
833 .join("637")
834 .join(LedgerStateSnapshot::IN_MEMORY_META),
835 PathBuf::from(LEDGER_DIR)
836 .join("637")
837 .join(LedgerStateSnapshot::IN_MEMORY_STATE),
838 PathBuf::from(LEDGER_DIR)
839 .join("637")
840 .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
841 .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
842 PathBuf::from(LEDGER_DIR)
843 .join("737")
844 .join(LedgerStateSnapshot::IN_MEMORY_META),
845 PathBuf::from(LEDGER_DIR)
846 .join("737")
847 .join(LedgerStateSnapshot::IN_MEMORY_STATE),
848 PathBuf::from(LEDGER_DIR)
849 .join("737")
850 .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
851 .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
852 ],
853 manifest.files()
854 );
855 assert_eq!(
856 Some(
857 fake_keys::signable_manifest_signature()[0]
858 .try_into()
859 .unwrap()
860 ),
861 manifest.signature()
862 )
863 }
864 }
865
866 mod compute_immutable_total_and_average_uncompressed_size {
867 use mithril_common::current_function;
868
869 use super::*;
870
871 #[tokio::test]
872 async fn should_compute_the_total_size_of_the_immutables() {
873 let test_dir = temp_dir_create!();
874 let immutable_trio_file_size = 777;
875
876 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
877 .with_immutables(&[1, 2, 3])
878 .set_immutable_trio_file_size(immutable_trio_file_size)
879 .with_legacy_ledger_snapshots(&[737])
880 .set_ledger_file_size(6666)
881 .with_volatile_files(&["blocks-0.dat"])
882 .set_volatile_file_size(99)
883 .build();
884
885 let snapshotter =
886 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
887
888 let sizes = snapshotter
889 .compute_immutable_files_total_uncompressed_size(2)
890 .await
891 .unwrap();
892
893 assert_eq!(immutable_trio_file_size * 2, sizes)
894 }
895
896 #[tokio::test]
897 async fn should_return_an_error_when_compute_up_to_immutable_0() {
898 let test_dir = temp_dir_create!();
899 let cardano_db = DummyCardanoDbBuilder::new(current_function!()).build();
900
901 let snapshotter =
902 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
903
904 snapshotter
905 .compute_immutable_files_total_uncompressed_size(0)
906 .await
907 .expect_err("Should return an error when no immutable file number");
908 }
909 }
910}