1use anyhow::{Context, anyhow};
2use async_trait::async_trait;
3use slog::{Logger, debug, warn};
4use std::fs;
5use std::path::{Path, PathBuf};
6use std::sync::Arc;
7
8use mithril_cardano_node_internal_database::entities::AncillaryFilesManifest;
9use mithril_cardano_node_internal_database::entities::{ImmutableFile, LedgerStateSnapshot};
10use mithril_cardano_node_internal_database::{IMMUTABLE_DIR, LEDGER_DIR, immutable_trio_names};
11use mithril_common::StdResult;
12use mithril_common::entities::{CompressionAlgorithm, ImmutableFileNumber};
13use mithril_common::logging::LoggerExtensions;
14
15use crate::dependency_injection::DependenciesBuilderError;
16use crate::tools::file_archiver::appender::{AppenderData, AppenderEntries, TarAppender};
17use crate::tools::file_archiver::{ArchiveParameters, FileArchive, FileArchiver};
18use crate::tools::file_size;
19
20use super::{Snapshotter, ancillary_signer::AncillarySigner};
21
22pub struct CompressedArchiveSnapshotter {
24 db_directory: PathBuf,
26
27 ongoing_snapshot_directory: PathBuf,
29
30 compression_algorithm: CompressionAlgorithm,
32
33 file_archiver: Arc<FileArchiver>,
34
35 ancillary_signer: Arc<dyn AncillarySigner>,
36
37 logger: Logger,
38}
39
40#[async_trait]
41impl Snapshotter for CompressedArchiveSnapshotter {
42 async fn snapshot_all_completed_immutables(
43 &self,
44 archive_name_without_extension: &str,
45 ) -> StdResult<FileArchive> {
46 debug!(
47 self.logger,
48 "Snapshotting all completed immutables into archive: '{archive_name_without_extension}'"
49 );
50
51 let paths_to_include = ImmutableFile::list_completed_in_dir(&self.db_directory)
52 .with_context(|| {
53 format!(
54 "Can not list completed immutables in database directory: '{}'",
55 self.db_directory.display()
56 )
57 })?
58 .into_iter()
59 .map(|immutable_file: ImmutableFile| {
60 PathBuf::from(IMMUTABLE_DIR).join(immutable_file.filename)
61 })
62 .collect();
63 let appender = AppenderEntries::new(paths_to_include, self.db_directory.clone())?;
64 self.snapshot(archive_name_without_extension, appender).await
65 }
66
67 async fn snapshot_ancillary(
68 &self,
69 immutable_file_number: ImmutableFileNumber,
70 archive_name_without_extension: &str,
71 ) -> StdResult<FileArchive> {
72 debug!(
73 self.logger,
74 "Snapshotting ancillary archive: '{archive_name_without_extension}'"
75 );
76
77 let temp_snapshot_directory =
78 self.temp_ancillary_snapshot_directory(archive_name_without_extension);
79 fs::create_dir(&temp_snapshot_directory).with_context(|| {
80 format!(
81 "Can not create temporary snapshot ancillary directory: '{}'",
82 temp_snapshot_directory.display()
83 )
84 })?;
85
86 let snapshot_result = self
87 .snapshot_ancillary_from_temp_directory(
88 immutable_file_number,
89 &temp_snapshot_directory,
90 archive_name_without_extension,
91 )
92 .await;
93
94 if let Err(e) = fs::remove_dir_all(&temp_snapshot_directory) {
95 warn!(
96 self.logger, "Failed to remove temporary snapshot ancillary directory '{}'", temp_snapshot_directory.display();
97 "error" => ?e
98 );
99 }
100 snapshot_result
101 }
102
103 async fn snapshot_immutable_trio(
104 &self,
105 immutable_file_number: ImmutableFileNumber,
106 archive_name_without_extension: &str,
107 ) -> StdResult<FileArchive> {
108 debug!(
109 self.logger,
110 "Snapshotting immutable trio {immutable_file_number} into archive '{archive_name_without_extension}'"
111 );
112
113 let files_to_archive = immutable_trio_names(immutable_file_number)
114 .iter()
115 .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
116 .collect();
117 let appender = AppenderEntries::new(files_to_archive, self.db_directory.clone())?;
118
119 self.snapshot(archive_name_without_extension, appender).await
120 }
121
122 async fn compute_immutable_files_total_uncompressed_size(
123 &self,
124 up_to_immutable_file_number: ImmutableFileNumber,
125 ) -> StdResult<u64> {
126 if up_to_immutable_file_number == 0 {
127 return Err(anyhow!(
128 "Could not compute the total size without immutable files"
129 ));
130 }
131 let immutable_directory = self.db_directory.join(IMMUTABLE_DIR);
132
133 tokio::task::spawn_blocking(move || -> StdResult<_> {
134 let immutable_paths = (1..=up_to_immutable_file_number)
135 .flat_map(immutable_trio_names)
136 .map(|filename| immutable_directory.join(filename))
137 .collect();
138
139 file_size::compute_size(immutable_paths)
140 })
141 .await?
142 }
143
144 fn compression_algorithm(&self) -> CompressionAlgorithm {
145 self.compression_algorithm
146 }
147}
148
149impl CompressedArchiveSnapshotter {
150 pub fn new(
152 db_directory: PathBuf,
153 ongoing_snapshot_directory: PathBuf,
154 compression_algorithm: CompressionAlgorithm,
155 file_archiver: Arc<FileArchiver>,
156 ancillary_signer: Arc<dyn AncillarySigner>,
157 logger: Logger,
158 ) -> StdResult<CompressedArchiveSnapshotter> {
159 if ongoing_snapshot_directory.exists() {
160 fs::remove_dir_all(&ongoing_snapshot_directory).with_context(|| {
161 format!(
162 "Can not remove snapshotter directory: '{}'.",
163 ongoing_snapshot_directory.display()
164 )
165 })?;
166 }
167
168 fs::create_dir(&ongoing_snapshot_directory).map_err(|e| {
169 DependenciesBuilderError::Initialization {
170 message: format!(
171 "Can not create snapshotter directory: '{}'.",
172 ongoing_snapshot_directory.display()
173 ),
174 error: Some(e.into()),
175 }
176 })?;
177
178 Ok(Self {
179 db_directory,
180 ongoing_snapshot_directory,
181 compression_algorithm,
182 file_archiver,
183 ancillary_signer,
184 logger: logger.new_with_component_name::<Self>(),
185 })
186 }
187
188 async fn snapshot<T: TarAppender + 'static>(
189 &self,
190 name_without_extension: &str,
191 appender: T,
192 ) -> StdResult<FileArchive> {
193 let file_archiver = self.file_archiver.clone();
194 let parameters = ArchiveParameters {
195 archive_name_without_extension: name_without_extension.to_string(),
196 target_directory: self.ongoing_snapshot_directory.clone(),
197 compression_algorithm: self.compression_algorithm,
198 };
199
200 let file_archive = tokio::task::spawn_blocking(move || -> StdResult<FileArchive> {
202 file_archiver.archive(parameters, appender)
203 })
204 .await??;
205
206 Ok(file_archive)
207 }
208
209 fn temp_ancillary_snapshot_directory(&self, discriminator: &str) -> PathBuf {
210 self.ongoing_snapshot_directory
211 .join(format!("temp-ancillary-{discriminator}"))
212 }
213
214 async fn snapshot_ancillary_from_temp_directory(
215 &self,
216 immutable_file_number: ImmutableFileNumber,
217 temp_snapshot_directory: &Path,
218 archive_name_without_extension: &str,
219 ) -> StdResult<FileArchive> {
220 let paths_to_include = self
221 .get_files_and_directories_for_ancillary_snapshot(
222 immutable_file_number,
223 temp_snapshot_directory,
224 )
225 .await?;
226 let signed_manifest = self
227 .build_and_sign_ancillary_manifest(paths_to_include.clone(), temp_snapshot_directory)
228 .await?;
229 let appender =
230 AppenderEntries::new(paths_to_include, temp_snapshot_directory.to_path_buf())?.chain(
231 AppenderData::from_json(
232 PathBuf::from(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME),
233 &signed_manifest,
234 )?,
235 );
236 self.snapshot(archive_name_without_extension, appender).await
237 }
238
239 async fn get_files_and_directories_for_ancillary_snapshot(
247 &self,
248 immutable_file_number: u64,
249 target_folder: &Path,
250 ) -> StdResult<Vec<PathBuf>> {
251 let next_immutable_file_number = immutable_file_number + 1;
252 let mut files_to_snapshot: Vec<PathBuf> = immutable_trio_names(next_immutable_file_number)
253 .into_iter()
254 .map(|filename| PathBuf::from(IMMUTABLE_DIR).join(filename))
255 .collect();
256
257 let db_ledger_dir = self.db_directory.join(LEDGER_DIR);
258 let ledger_files = LedgerStateSnapshot::list_all_in_dir(&db_ledger_dir)?;
259 let latest_ledger_files: Vec<PathBuf> = ledger_files
260 .iter()
261 .rev()
262 .take(2)
263 .flat_map(|ledger_state_snapshot| ledger_state_snapshot.get_files_relative_path())
264 .map(|path| PathBuf::from(LEDGER_DIR).join(path))
265 .collect();
266 if latest_ledger_files.is_empty() {
267 return Err(anyhow!(
268 "No ledger state snapshot found in the ledger directory: '{}'",
269 db_ledger_dir.display()
270 ));
271 }
272 files_to_snapshot.extend(latest_ledger_files);
273
274 fs::create_dir(target_folder.join(IMMUTABLE_DIR))
275 .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
276 fs::create_dir(target_folder.join(LEDGER_DIR))
277 .with_context(|| format!("Can not create folder: `{}`", target_folder.display()))?;
278
279 for file in &files_to_snapshot {
280 if let Some(parent_dir) = file.parent() {
282 let target_parent_dir = target_folder.join(parent_dir);
283 if !target_parent_dir.exists() {
284 fs::create_dir_all(&target_parent_dir).with_context(|| {
285 format!("Can not create folder: `{}`", target_parent_dir.display())
286 })?;
287 }
288 }
289
290 let source = self.db_directory.join(file);
291 let target = target_folder.join(file);
292 tokio::fs::copy(&source, &target).await.with_context(|| {
293 format!(
294 "Failed to copy file `{}` to `{}`",
295 source.display(),
296 target.display()
297 )
298 })?;
299 }
300
301 Ok(files_to_snapshot)
302 }
303
304 async fn build_and_sign_ancillary_manifest(
305 &self,
306 paths_to_include: Vec<PathBuf>,
307 temp_snapshot_directory: &Path,
308 ) -> StdResult<AncillaryFilesManifest> {
309 let mut manifest =
310 AncillaryFilesManifest::from_paths(temp_snapshot_directory, paths_to_include).await?;
311 let signature = self
312 .ancillary_signer
313 .compute_ancillary_manifest_signature(&manifest)
314 .await?;
315 manifest.set_signature(signature);
316
317 Ok(manifest)
318 }
319}
320
321#[cfg(test)]
322mod tests {
323 use std::fs::File;
324 use std::path::Path;
325 use std::sync::Arc;
326
327 use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder;
328 use mithril_common::test_utils::assert_equivalent;
329 use mithril_common::{assert_dir_eq, current_function, temp_dir_create};
330
331 use crate::services::ancillary_signer::MockAncillarySigner;
332 use crate::test_tools::TestLogger;
333
334 use super::*;
335
336 fn list_files(test_dir: &Path) -> Vec<String> {
337 fs::read_dir(test_dir)
338 .unwrap()
339 .map(|f| f.unwrap().file_name().to_str().unwrap().to_owned())
340 .collect()
341 }
342
343 fn snapshotter_for_test(
344 test_directory: &Path,
345 db_directory: &Path,
346 compression_algorithm: CompressionAlgorithm,
347 ) -> CompressedArchiveSnapshotter {
348 CompressedArchiveSnapshotter::new(
349 db_directory.to_path_buf(),
350 test_directory.join("ongoing_snapshot"),
351 compression_algorithm,
352 Arc::new(FileArchiver::new_for_test(
353 test_directory.join("verification"),
354 )),
355 Arc::new(MockAncillarySigner::new()),
356 TestLogger::stdout(),
357 )
358 .unwrap()
359 }
360
361 #[test]
362 fn return_parametrized_compression_algorithm() {
363 let test_dir = temp_dir_create!();
364 let snapshotter = snapshotter_for_test(
365 &test_dir,
366 Path::new("whatever"),
367 CompressionAlgorithm::Zstandard,
368 );
369
370 assert_eq!(
371 CompressionAlgorithm::Zstandard,
372 snapshotter.compression_algorithm()
373 );
374 }
375
376 #[test]
377 fn should_create_directory_if_does_not_exist() {
378 let test_dir = temp_dir_create!();
379 let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
380 let db_directory = test_dir.join("whatever");
381
382 CompressedArchiveSnapshotter::new(
383 db_directory,
384 ongoing_snapshot_directory.clone(),
385 CompressionAlgorithm::Gzip,
386 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
387 Arc::new(MockAncillarySigner::new()),
388 TestLogger::stdout(),
389 )
390 .unwrap();
391
392 assert!(ongoing_snapshot_directory.is_dir());
393 }
394
395 #[test]
396 fn should_clean_ongoing_snapshot_directory_if_already_exists() {
397 let test_dir = temp_dir_create!();
398 let ongoing_snapshot_directory = test_dir.join("ongoing_snapshot");
399 let db_directory = test_dir.join("whatever");
400
401 fs::create_dir_all(&ongoing_snapshot_directory).unwrap();
402
403 File::create(ongoing_snapshot_directory.join("whatever.txt")).unwrap();
404
405 CompressedArchiveSnapshotter::new(
406 db_directory,
407 ongoing_snapshot_directory.clone(),
408 CompressionAlgorithm::Gzip,
409 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
410 Arc::new(MockAncillarySigner::new()),
411 TestLogger::stdout(),
412 )
413 .unwrap();
414
415 assert_eq!(0, fs::read_dir(ongoing_snapshot_directory).unwrap().count());
416 }
417
418 #[tokio::test]
419 async fn should_create_snapshots_in_its_ongoing_snapshot_directory() {
420 let test_dir = temp_dir_create!();
421 let pending_snapshot_directory = test_dir.join("pending_snapshot");
422 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
423 .with_immutables(&[1])
424 .append_immutable_trio()
425 .build();
426
427 let snapshotter = CompressedArchiveSnapshotter::new(
428 cardano_db.get_dir().clone(),
429 pending_snapshot_directory.clone(),
430 CompressionAlgorithm::Gzip,
431 Arc::new(FileArchiver::new_for_test(test_dir.join("verification"))),
432 Arc::new(MockAncillarySigner::new()),
433 TestLogger::stdout(),
434 )
435 .unwrap();
436 let snapshot = snapshotter
437 .snapshot_all_completed_immutables("whatever")
438 .await
439 .unwrap();
440
441 assert_eq!(
442 pending_snapshot_directory,
443 snapshot.get_file_path().parent().unwrap()
444 );
445 }
446
447 mod snapshot_all_completed_immutables {
448 use super::*;
449
450 #[tokio::test]
451 async fn include_only_completed_immutables() {
452 let test_dir = temp_dir_create!();
453 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
454 .with_immutables(&[1, 2, 3])
455 .append_immutable_trio()
456 .with_legacy_ledger_snapshots(&[437])
457 .with_volatile_files(&["blocks-0.dat"])
458 .with_non_immutables(&["random_file.txt", "00002.trap"])
459 .build();
460
461 let snapshotter =
462 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
463
464 let snapshot = snapshotter
465 .snapshot_all_completed_immutables("completed_immutables")
466 .await
467 .unwrap();
468
469 let unpack_dir = snapshot.unpack_gzip(&test_dir);
470 let unpacked_files = list_files(&unpack_dir);
471 let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
472
473 assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
474 assert_equivalent(
475 vec![
476 "00001.chunk".to_string(),
477 "00001.primary".to_string(),
478 "00001.secondary".to_string(),
479 "00002.chunk".to_string(),
480 "00002.primary".to_string(),
481 "00002.secondary".to_string(),
482 "00003.chunk".to_string(),
483 "00003.primary".to_string(),
484 "00003.secondary".to_string(),
485 ],
486 unpacked_immutable_files,
487 );
488 }
489 }
490
491 mod snapshot_immutable_trio {
492 use super::*;
493
494 #[tokio::test]
495 async fn include_only_immutable_trio() {
496 let test_dir = temp_dir_create!();
497 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
498 .with_immutables(&[1, 2, 3])
499 .with_legacy_ledger_snapshots(&[437])
500 .with_volatile_files(&["blocks-0.dat"])
501 .with_non_immutables(&["random_file.txt", "00002.trap"])
502 .build();
503
504 let snapshotter =
505 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
506
507 let snapshot = snapshotter.snapshot_immutable_trio(2, "immutable-2").await.unwrap();
508
509 let unpack_dir = snapshot.unpack_gzip(&test_dir);
510 let unpacked_files = list_files(&unpack_dir);
511 let unpacked_immutable_files = list_files(&unpack_dir.join(IMMUTABLE_DIR));
512
513 assert_equivalent(vec![IMMUTABLE_DIR.to_string()], unpacked_files);
514 assert_equivalent(
515 vec![
516 "00002.chunk".to_string(),
517 "00002.primary".to_string(),
518 "00002.secondary".to_string(),
519 ],
520 unpacked_immutable_files,
521 );
522 }
523 }
524
525 mod snapshot_ancillary {
526 use mithril_common::test_utils::fake_keys;
527
528 use super::*;
529
530 #[tokio::test]
531 async fn getting_files_to_include_for_legacy_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure()
532 {
533 let test_dir = temp_dir_create!();
534 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
535 .with_immutables(&[1, 2])
536 .with_legacy_ledger_snapshots(&[737])
537 .build();
538 let snapshotter =
539 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
540 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
541 fs::create_dir(&ancillary_snapshot_dir).unwrap();
542
543 snapshotter
544 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
545 .await
546 .unwrap();
547
548 assert_dir_eq!(
549 &ancillary_snapshot_dir,
550 format!(
551 "* {IMMUTABLE_DIR}/
552 ** 00002.chunk
553 ** 00002.primary
554 ** 00002.secondary
555 * {LEDGER_DIR}/
556 ** 737"
557 )
558 );
559 }
560
561 #[tokio::test]
562 async fn getting_files_to_include_for_in_memory_ledger_snapshot_copy_them_to_a_target_directory_while_keeping_source_dir_structure()
563 {
564 let test_dir = temp_dir_create!();
565 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
566 .with_immutables(&[1, 2])
567 .with_in_memory_ledger_snapshots(&[737])
568 .build();
569 let snapshotter =
570 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
571 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
572 fs::create_dir(&ancillary_snapshot_dir).unwrap();
573
574 snapshotter
575 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
576 .await
577 .unwrap();
578
579 assert_dir_eq!(
580 &ancillary_snapshot_dir,
581 format!(
582 "* {IMMUTABLE_DIR}/
583 ** 00002.chunk
584 ** 00002.primary
585 ** 00002.secondary
586 * {LEDGER_DIR}/
587 ** 737/
588 *** {}/
589 **** {}
590 *** {}
591 *** {}",
592 LedgerStateSnapshot::IN_MEMORY_TABLES,
593 LedgerStateSnapshot::IN_MEMORY_TVAR,
594 LedgerStateSnapshot::IN_MEMORY_META,
595 LedgerStateSnapshot::IN_MEMORY_STATE,
596 )
597 );
598 }
599
600 #[tokio::test]
601 async fn getting_files_to_include_fails_when_no_ledger_file_found() {
602 let test_dir = temp_dir_create!();
603 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
604 .with_immutables(&[1, 2])
605 .build();
606 let snapshotter =
607 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
608 let ancillary_snapshot_dir = test_dir.join("ancillary_snapshot");
609 fs::create_dir(&ancillary_snapshot_dir).unwrap();
610
611 snapshotter
612 .get_files_and_directories_for_ancillary_snapshot(1, &ancillary_snapshot_dir)
613 .await
614 .expect_err("Should fail if no ledger file found");
615 }
616
617 #[tokio::test]
618 async fn delete_temporary_working_directory_after_snapshot_is_created() {
619 let test_dir = temp_dir_create!();
620 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
621 .with_immutables(&[1, 2])
622 .with_legacy_ledger_snapshots(&[637])
623 .build();
624 let snapshotter = CompressedArchiveSnapshotter {
625 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
626 fake_keys::signable_manifest_signature()[0],
627 )),
628 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
629 };
630
631 snapshotter.snapshot_ancillary(1, "ancillary").await.unwrap();
632
633 let temp_ancillary_snapshot_dir =
634 snapshotter.temp_ancillary_snapshot_directory("ancillary");
635 assert!(
636 !temp_ancillary_snapshot_dir.exists(),
637 "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
638 temp_ancillary_snapshot_dir.display()
639 );
640 }
641
642 #[tokio::test]
643 async fn delete_temporary_working_directory_even_if_snapshot_creation_fails() {
644 let test_dir = temp_dir_create!();
645 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
646 .with_immutables(&[1, 2])
647 .with_legacy_ledger_snapshots(&[637])
648 .build();
649 let snapshotter = CompressedArchiveSnapshotter {
650 ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message("failure")),
651 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
652 };
653
654 snapshotter.snapshot_ancillary(1, "ancillary").await.unwrap_err();
655
656 let temp_ancillary_snapshot_dir =
657 snapshotter.temp_ancillary_snapshot_directory("ancillary");
658 assert!(
659 !temp_ancillary_snapshot_dir.exists(),
660 "Expected temporary ancillary snapshot directory to be deleted, but it still exists: {}",
661 temp_ancillary_snapshot_dir.display()
662 );
663 }
664
665 #[tokio::test]
666 async fn create_archive_should_embed_only_two_last_ledgers_and_last_immutables() {
667 let test_dir = temp_dir_create!();
668 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
669 .with_immutables(&[1, 2, 3])
670 .with_legacy_ledger_snapshots(&[437, 537, 637, 737])
671 .with_non_ledger_files(&["9not_included"])
672 .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
673 .build();
674 fs::create_dir(cardano_db.get_dir().join("whatever")).unwrap();
675
676 let snapshotter = CompressedArchiveSnapshotter {
677 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
678 fake_keys::signable_manifest_signature()[0],
679 )),
680 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
681 };
682
683 let snapshot = snapshotter.snapshot_ancillary(2, "ancillary").await.unwrap();
684
685 let unpack_dir = snapshot.unpack_gzip(&test_dir);
686 assert_dir_eq!(
687 &unpack_dir,
688 format!(
690 "* {IMMUTABLE_DIR}/
691 ** 00003.chunk
692 ** 00003.primary
693 ** 00003.secondary
694 * {LEDGER_DIR}/
695 ** 637
696 ** 737
697 * {}",
698 AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME
699 )
700 );
701 }
702
703 #[tokio::test]
704 async fn create_archive_fail_if_manifest_signing_fail() {
705 let test_dir = temp_dir_create!();
706 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
707 .with_immutables(&[1, 2])
708 .with_legacy_ledger_snapshots(&[737])
709 .build();
710
711 let snapshotter = CompressedArchiveSnapshotter {
712 ancillary_signer: Arc::new(MockAncillarySigner::that_fails_with_message(
713 "MockAncillarySigner failed",
714 )),
715 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
716 };
717
718 let err = snapshotter
719 .snapshot_ancillary(1, "ancillary")
720 .await
721 .expect_err("Must fail if manifest signing fails");
722 assert!(
723 err.to_string().contains("MockAncillarySigner failed"),
724 "Expected error message to be raised by the mock ancillary signer, but got: '{err:?}'",
725 );
726 }
727
728 #[tokio::test]
729 async fn create_archive_of_legacy_ledger_snapshot_generate_sign_and_include_manifest_file()
730 {
731 let test_dir = temp_dir_create!();
732 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
733 .with_immutables(&[1, 2, 3])
734 .with_legacy_ledger_snapshots(&[537, 637, 737])
735 .with_non_immutables(&["not_to_include.txt"])
736 .build();
737 File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
738
739 let snapshotter = CompressedArchiveSnapshotter {
740 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
741 fake_keys::signable_manifest_signature()[0],
742 )),
743 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
744 };
745
746 let archive = snapshotter.snapshot_ancillary(2, "ancillary").await.unwrap();
747 let unpacked = archive.unpack_gzip(test_dir);
748 let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
749
750 assert!(manifest_path.exists());
751
752 let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
753 File::open(&manifest_path).unwrap(),
754 )
755 .unwrap();
756
757 assert_eq!(
758 vec![
759 PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
760 PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
761 PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
762 PathBuf::from(LEDGER_DIR).join("637"),
763 PathBuf::from(LEDGER_DIR).join("737"),
764 ],
765 manifest.files()
766 );
767 assert_eq!(
768 Some(fake_keys::signable_manifest_signature()[0].try_into().unwrap()),
769 manifest.signature()
770 )
771 }
772
773 #[tokio::test]
774 async fn create_archive_of_in_memory_ledger_snapshot_generate_sign_and_include_manifest_file()
775 {
776 let test_dir = temp_dir_create!();
777 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
778 .with_immutables(&[1, 2, 3])
779 .with_in_memory_ledger_snapshots(&[537, 637, 737])
780 .with_non_immutables(&["not_to_include.txt"])
781 .build();
782 File::create(cardano_db.get_dir().join("not_to_include_as_well.txt")).unwrap();
783
784 let snapshotter = CompressedArchiveSnapshotter {
785 ancillary_signer: Arc::new(MockAncillarySigner::that_succeeds_with_signature(
786 fake_keys::signable_manifest_signature()[0],
787 )),
788 ..snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip)
789 };
790
791 let archive = snapshotter.snapshot_ancillary(2, "ancillary").await.unwrap();
792 let unpacked = archive.unpack_gzip(test_dir);
793 let manifest_path = unpacked.join(AncillaryFilesManifest::ANCILLARY_MANIFEST_FILE_NAME);
794
795 assert!(manifest_path.exists());
796
797 let manifest = serde_json::from_reader::<_, AncillaryFilesManifest>(
798 File::open(&manifest_path).unwrap(),
799 )
800 .unwrap();
801
802 assert_eq!(
803 vec![
804 PathBuf::from(IMMUTABLE_DIR).join("00003.chunk"),
805 PathBuf::from(IMMUTABLE_DIR).join("00003.primary"),
806 PathBuf::from(IMMUTABLE_DIR).join("00003.secondary"),
807 PathBuf::from(LEDGER_DIR)
808 .join("637")
809 .join(LedgerStateSnapshot::IN_MEMORY_META),
810 PathBuf::from(LEDGER_DIR)
811 .join("637")
812 .join(LedgerStateSnapshot::IN_MEMORY_STATE),
813 PathBuf::from(LEDGER_DIR)
814 .join("637")
815 .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
816 .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
817 PathBuf::from(LEDGER_DIR)
818 .join("737")
819 .join(LedgerStateSnapshot::IN_MEMORY_META),
820 PathBuf::from(LEDGER_DIR)
821 .join("737")
822 .join(LedgerStateSnapshot::IN_MEMORY_STATE),
823 PathBuf::from(LEDGER_DIR)
824 .join("737")
825 .join(LedgerStateSnapshot::IN_MEMORY_TABLES)
826 .join(LedgerStateSnapshot::IN_MEMORY_TVAR),
827 ],
828 manifest.files()
829 );
830 assert_eq!(
831 Some(fake_keys::signable_manifest_signature()[0].try_into().unwrap()),
832 manifest.signature()
833 )
834 }
835 }
836
837 mod compute_immutable_total_and_average_uncompressed_size {
838 use mithril_common::current_function;
839
840 use super::*;
841
842 #[tokio::test]
843 async fn should_compute_the_total_size_of_the_immutables() {
844 let test_dir = temp_dir_create!();
845 let immutable_trio_file_size = 777;
846
847 let cardano_db = DummyCardanoDbBuilder::new(current_function!())
848 .with_immutables(&[1, 2, 3])
849 .set_immutable_trio_file_size(immutable_trio_file_size)
850 .with_legacy_ledger_snapshots(&[737])
851 .set_ledger_file_size(6666)
852 .with_volatile_files(&["blocks-0.dat"])
853 .set_volatile_file_size(99)
854 .build();
855
856 let snapshotter =
857 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
858
859 let sizes = snapshotter
860 .compute_immutable_files_total_uncompressed_size(2)
861 .await
862 .unwrap();
863
864 assert_eq!(immutable_trio_file_size * 2, sizes)
865 }
866
867 #[tokio::test]
868 async fn should_return_an_error_when_compute_up_to_immutable_0() {
869 let test_dir = temp_dir_create!();
870 let cardano_db = DummyCardanoDbBuilder::new(current_function!()).build();
871
872 let snapshotter =
873 snapshotter_for_test(&test_dir, cardano_db.get_dir(), CompressionAlgorithm::Gzip);
874
875 snapshotter
876 .compute_immutable_files_total_uncompressed_size(0)
877 .await
878 .expect_err("Should return an error when no immutable file number");
879 }
880 }
881}