1use std::{
2 fs,
3 path::{Path, PathBuf},
4 sync::Arc,
5};
6
7use anyhow::{anyhow, Context};
8use async_trait::async_trait;
9use regex::Regex;
10use slog::{error, Logger};
11
12use mithril_common::{
13 digesters::immutable_trio_names,
14 entities::{CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation, MultiFilesUri},
15 logging::LoggerExtensions,
16 StdResult,
17};
18
19use crate::{
20 file_uploaders::{GcpUploader, LocalUploader},
21 services::Snapshotter,
22 tools::file_size,
23 DumbUploader, FileUploader,
24};
25
26fn immmutable_file_number_extractor(file_uri: &str) -> StdResult<Option<String>> {
27 let regex = Regex::new(r".*(\d{5})")?;
28
29 Ok(regex
30 .captures(file_uri)
31 .and_then(|mat| mat.get(1))
32 .map(|immutable_match| {
33 let mut template = file_uri.to_string();
34 template.replace_range(immutable_match.range(), "{immutable_file_number}");
35
36 template
37 }))
38}
39
40#[cfg_attr(test, mockall::automock)]
42#[async_trait]
43pub trait ImmutableFilesUploader: Send + Sync {
44 async fn batch_upload(
46 &self,
47 filepaths: &[PathBuf],
48 compression_algorithm: Option<CompressionAlgorithm>,
49 ) -> StdResult<ImmutablesLocation>;
50}
51
52#[derive(Debug)]
53pub struct ImmutablesUpload {
54 pub locations: Vec<ImmutablesLocation>,
55 pub average_size: u64,
56 pub total_size: u64,
57}
58
59#[async_trait]
60impl ImmutableFilesUploader for DumbUploader {
61 async fn batch_upload(
62 &self,
63 filepaths: &[PathBuf],
64 compression_algorithm: Option<CompressionAlgorithm>,
65 ) -> StdResult<ImmutablesLocation> {
66 let last_file_path = filepaths.last().ok_or_else(|| {
67 anyhow!("No file to upload with 'DumbUploader' as the filepaths list is empty")
68 })?;
69
70 let template_uri = MultiFilesUri::extract_template_from_uris(
71 vec![self.upload(last_file_path).await?.into()],
72 immmutable_file_number_extractor,
73 )?
74 .ok_or_else(|| {
75 anyhow!("No matching template found in the uploaded files with 'DumbUploader'")
76 })?;
77
78 Ok(ImmutablesLocation::CloudStorage {
79 uri: MultiFilesUri::Template(template_uri),
80 compression_algorithm,
81 })
82 }
83}
84
85#[async_trait]
86impl ImmutableFilesUploader for LocalUploader {
87 async fn batch_upload(
88 &self,
89 filepaths: &[PathBuf],
90 compression_algorithm: Option<CompressionAlgorithm>,
91 ) -> StdResult<ImmutablesLocation> {
92 let mut file_uris = Vec::new();
93 for filepath in filepaths {
94 file_uris.push(self.upload(filepath).await?.into());
95 }
96
97 let template_uri =
98 MultiFilesUri::extract_template_from_uris(file_uris, immmutable_file_number_extractor)?
99 .ok_or_else(|| {
100 anyhow!("No matching template found in the uploaded files with 'LocalUploader'")
101 })?;
102
103 Ok(ImmutablesLocation::CloudStorage {
104 uri: MultiFilesUri::Template(template_uri),
105 compression_algorithm,
106 })
107 }
108}
109
110#[async_trait]
111impl ImmutableFilesUploader for GcpUploader {
112 async fn batch_upload(
113 &self,
114 filepaths: &[PathBuf],
115 compression_algorithm: Option<CompressionAlgorithm>,
116 ) -> StdResult<ImmutablesLocation> {
117 let mut file_uris = Vec::new();
118 for filepath in filepaths {
119 file_uris.push(self.upload(filepath).await?.into());
120 }
121
122 let template_uri =
123 MultiFilesUri::extract_template_from_uris(file_uris, immmutable_file_number_extractor)?
124 .ok_or_else(|| {
125 anyhow!("No matching template found in the uploaded files with 'GcpUploader'")
126 })?;
127
128 Ok(ImmutablesLocation::CloudStorage {
129 uri: MultiFilesUri::Template(template_uri),
130 compression_algorithm,
131 })
132 }
133}
134
135pub struct ImmutableArtifactBuilder {
136 immutables_storage_dir: PathBuf,
137 uploaders: Vec<Arc<dyn ImmutableFilesUploader>>,
138 snapshotter: Arc<dyn Snapshotter>,
139 logger: Logger,
140}
141
142impl ImmutableArtifactBuilder {
143 pub fn new(
144 immutables_storage_dir: PathBuf,
145 uploaders: Vec<Arc<dyn ImmutableFilesUploader>>,
146 snapshotter: Arc<dyn Snapshotter>,
147 logger: Logger,
148 ) -> StdResult<Self> {
149 if uploaders.is_empty() {
150 return Err(anyhow!(
151 "At least one uploader is required to create an 'ImmutableArtifactBuilder'"
152 ));
153 }
154
155 if !immutables_storage_dir.exists() {
156 fs::create_dir(&immutables_storage_dir).with_context(|| {
157 format!(
158 "Can not create immutable storage directory: '{}'",
159 immutables_storage_dir.display()
160 )
161 })?;
162 }
163
164 Ok(Self {
165 immutables_storage_dir,
166 uploaders,
167 snapshotter,
168 logger: logger.new_with_component_name::<Self>(),
169 })
170 }
171
172 pub async fn upload(
173 &self,
174 up_to_immutable_file_number: ImmutableFileNumber,
175 ) -> StdResult<ImmutablesUpload> {
176 let (archives_paths, compression_algorithm) = self
177 .immutable_archives_paths_creating_the_missing_ones(up_to_immutable_file_number)
178 .await?;
179 let locations = self
180 .upload_immutable_archives(&archives_paths, compression_algorithm)
181 .await?;
182
183 let (total_size, average_size) = {
184 let db_directory = self.immutables_storage_dir.clone();
185 let immutable_file_number = up_to_immutable_file_number;
186 tokio::task::spawn_blocking(move || -> StdResult<_> {
187 Self::compute_total_and_average_uncompressed_size(
188 &db_directory,
189 immutable_file_number,
190 )
191 })
192 .await??
193 };
194
195 Ok(ImmutablesUpload {
196 locations,
197 average_size,
198 total_size,
199 })
200 }
201
202 pub async fn immutable_archives_paths_creating_the_missing_ones(
203 &self,
204 up_to_immutable_file_number: ImmutableFileNumber,
205 ) -> StdResult<(Vec<PathBuf>, CompressionAlgorithm)> {
206 let mut archive_paths = vec![];
207 let compression_algorithm = self.snapshotter.compression_algorithm();
208
209 for immutable_file_number in 1..=up_to_immutable_file_number {
210 let archive_name_without_extension = format!("{immutable_file_number:05}");
211 let archive_name = format!(
212 "{archive_name_without_extension}.{}",
213 compression_algorithm.tar_file_extension()
214 );
215
216 if let Some(existing_archive) = self.retrieve_existing_snapshot_archive(&archive_name) {
217 archive_paths.push(existing_archive);
218 } else {
219 let snapshot = self
220 .snapshotter
221 .snapshot_immutable_trio(immutable_file_number, &archive_name_without_extension)
222 .await?;
223
224 let target_path = self.immutables_storage_dir.join(&archive_name);
225 fs::rename(snapshot.get_file_path(), &target_path).with_context(|| {
226 format!(
227 "Can not move archive of immutable {immutable_file_number} from '{}' to '{}'",
228 snapshot.get_file_path().display(),
229 target_path.display()
230 )
231 })?;
232
233 archive_paths.push(target_path);
234 }
235 }
236
237 Ok((archive_paths, compression_algorithm))
238 }
239
240 async fn upload_immutable_archives(
241 &self,
242 archive_paths: &[PathBuf],
243 compression_algorithm: CompressionAlgorithm,
244 ) -> StdResult<Vec<ImmutablesLocation>> {
245 let mut locations = Vec::new();
246 for uploader in &self.uploaders {
247 let result = uploader
248 .batch_upload(archive_paths, Some(compression_algorithm))
249 .await;
250 match result {
251 Ok(location) => {
252 locations.push(location);
253 }
254 Err(e) => {
255 error!(
256 self.logger,
257 "Failed to upload immutable archive";
258 "error" => e.to_string()
259 );
260 }
261 }
262 }
263
264 if locations.is_empty() {
265 return Err(anyhow!(
266 "Failed to upload immutable archive with all uploaders"
267 ));
268 }
269
270 Ok(locations)
271 }
272
273 fn retrieve_existing_snapshot_archive(&self, expected_archive_name: &str) -> Option<PathBuf> {
274 let expected_archive_path = self.immutables_storage_dir.join(expected_archive_name);
275 expected_archive_path
276 .exists()
277 .then_some(expected_archive_path)
278 }
279
280 fn compute_total_and_average_uncompressed_size(
284 immutable_directory: &Path,
285 up_to_immutable_file_number: ImmutableFileNumber,
286 ) -> StdResult<(u64, u64)> {
287 if up_to_immutable_file_number == 0 {
288 return Err(anyhow!(
289 "Could not compute the total and average size without immutable files"
290 ));
291 }
292
293 let immutable_paths = (1..=up_to_immutable_file_number)
294 .flat_map(immutable_trio_names)
295 .map(|filename| immutable_directory.join(filename))
296 .collect();
297
298 let total_size = file_size::compute_size(immutable_paths)?;
299
300 Ok((total_size, total_size / up_to_immutable_file_number))
301 }
302}
303
304#[cfg(test)]
305mod tests {
306 use mithril_common::{
307 current_function,
308 digesters::DummyCardanoDbBuilder,
309 entities::TemplateUri,
310 test_utils::{assert_equivalent, equivalent_to, TempDir},
311 };
312 use std::fs::File;
313 use std::io::Write;
314
315 use crate::services::ancillary_signer::MockAncillarySigner;
316 use crate::services::{CompressedArchiveSnapshotter, DumbSnapshotter, MockSnapshotter};
317 use crate::test_tools::TestLogger;
318 use crate::tools::file_archiver::FileArchiver;
319
320 use super::*;
321
322 fn fake_uploader(
323 archive_paths: Vec<&str>,
324 location_uri: &str,
325 compression_algorithm: Option<CompressionAlgorithm>,
326 ) -> MockImmutableFilesUploader {
327 let uri = location_uri.to_string();
328 let archive_paths: Vec<_> = archive_paths.into_iter().map(String::from).collect();
329
330 let mut uploader = MockImmutableFilesUploader::new();
331 uploader
332 .expect_batch_upload()
333 .withf(move |p, algorithm| {
334 let paths: Vec<_> = p.iter().map(|s| s.to_string_lossy().into_owned()).collect();
335
336 equivalent_to(paths, archive_paths.clone()) && algorithm == &compression_algorithm
337 })
338 .times(1)
339 .return_once(move |_, _| {
340 Ok(ImmutablesLocation::CloudStorage {
341 uri: MultiFilesUri::Template(TemplateUri(uri)),
342 compression_algorithm,
343 })
344 });
345
346 uploader
347 }
348
349 fn fake_uploader_returning_error() -> MockImmutableFilesUploader {
350 let mut uploader = MockImmutableFilesUploader::new();
351 uploader
352 .expect_batch_upload()
353 .return_once(|_, _| Err(anyhow!("Failure while uploading...")));
354
355 uploader
356 }
357
358 fn create_fake_file(path: &Path, content: &str) {
359 let mut file = File::create(path).unwrap();
360 write!(file, "{content}").unwrap();
361 }
362
363 macro_rules! assert_file_content {
364 ($path:expr, $expected_content:expr) => {
365 assert!($path.exists());
366 let content = std::fs::read_to_string(&$path).unwrap();
367 assert_eq!(content, $expected_content);
368 };
369 }
370
371 fn get_builder_work_dir<N: Into<String>>(test_name: N) -> PathBuf {
372 TempDir::create("cdb_immutable_builder", test_name)
373 }
374
375 #[tokio::test]
376 async fn upload_call_archive_creation_and_upload_to_retrieve_locations() {
377 let work_dir = get_builder_work_dir("upload_call_archive_creation_and_upload");
378 let test_dir = "upload_call_archive_creation_and_upload/cardano_database";
379 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
380 .with_immutables(&[1, 2])
381 .build();
382
383 let db_directory = cardano_db.get_dir().to_path_buf();
384 let snapshotter = CompressedArchiveSnapshotter::new(
385 db_directory.clone(),
386 db_directory.parent().unwrap().join("snapshot_dest"),
387 CompressionAlgorithm::Gzip,
388 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
389 Arc::new(MockAncillarySigner::new()),
390 TestLogger::stdout(),
391 )
392 .unwrap();
393
394 let uploader = fake_uploader(
395 vec![
396 work_dir.join("00001.tar.gz").to_str().unwrap(),
397 work_dir.join("00002.tar.gz").to_str().unwrap(),
398 ],
399 "archive.tar.gz",
400 Some(CompressionAlgorithm::Gzip),
401 );
402
403 let builder = ImmutableArtifactBuilder::new(
404 work_dir,
405 vec![Arc::new(uploader)],
406 Arc::new(snapshotter),
407 TestLogger::stdout(),
408 )
409 .unwrap();
410
411 let upload = builder.upload(2).await.unwrap();
412
413 assert_equivalent(
414 upload.locations,
415 vec![ImmutablesLocation::CloudStorage {
416 uri: MultiFilesUri::Template(TemplateUri("archive.tar.gz".to_string())),
417 compression_algorithm: Some(CompressionAlgorithm::Gzip),
418 }],
419 )
420 }
421
422 #[test]
423 fn create_immutable_builder_should_create_immutable_storage_dir_if_not_exist() {
424 let work_dir = get_builder_work_dir(
425 "create_immutable_builder_should_create_immutable_storage_dir_if_not_exist",
426 );
427 let immutable_storage_dir = work_dir.join("immutable");
428
429 assert!(!immutable_storage_dir.exists());
430
431 ImmutableArtifactBuilder::new(
432 immutable_storage_dir.clone(),
433 vec![Arc::new(DumbUploader::default())],
434 Arc::new(DumbSnapshotter::default()),
435 TestLogger::stdout(),
436 )
437 .unwrap();
438
439 assert!(immutable_storage_dir.exists());
440 }
441
442 #[test]
443 fn create_immutable_builder_should_not_create_or_remove_immutable_storage_dir_if_it_exist() {
444 let immutable_storage_dir = get_builder_work_dir(
445 "create_immutable_builder_should_not_create_or_remove_immutable_storage_dir_if_it_exist",
446 );
447 let existing_file_path = immutable_storage_dir.join("file.txt");
448 create_fake_file(&existing_file_path, "existing file content");
449
450 ImmutableArtifactBuilder::new(
451 immutable_storage_dir,
452 vec![Arc::new(DumbUploader::default())],
453 Arc::new(DumbSnapshotter::default()),
454 TestLogger::stdout(),
455 )
456 .unwrap();
457
458 assert_file_content!(existing_file_path, "existing file content");
459 }
460
461 mod create_archive {
462 use super::*;
463
464 #[tokio::test]
465 async fn snapshot_immutables_files_up_to_the_given_immutable_file_number() {
466 let work_dir = get_builder_work_dir(
467 "snapshot_immutables_files_up_to_the_given_immutable_file_number",
468 );
469 let test_dir =
470 "snapshot_immutables_files_up_to_the_given_immutable_file_number/cardano_database";
471 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
472 .with_immutables(&[1, 2])
473 .build();
474
475 let db_directory = cardano_db.get_dir().to_path_buf();
476 let snapshotter = CompressedArchiveSnapshotter::new(
477 db_directory.clone(),
478 db_directory.parent().unwrap().join("snapshot_dest"),
479 CompressionAlgorithm::Gzip,
480 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
481 Arc::new(MockAncillarySigner::new()),
482 TestLogger::stdout(),
483 )
484 .unwrap();
485
486 let builder = ImmutableArtifactBuilder::new(
487 work_dir.clone(),
488 vec![Arc::new(MockImmutableFilesUploader::new())],
489 Arc::new(snapshotter),
490 TestLogger::stdout(),
491 )
492 .unwrap();
493
494 let (archive_paths, _) = builder
495 .immutable_archives_paths_creating_the_missing_ones(2)
496 .await
497 .unwrap();
498
499 assert_equivalent(
500 archive_paths,
501 vec![work_dir.join("00001.tar.gz"), work_dir.join("00002.tar.gz")],
502 )
503 }
504
505 #[tokio::test]
506 async fn return_error_when_one_of_the_three_immutable_files_is_missing() {
507 let work_dir = get_builder_work_dir(
508 "return_error_when_one_of_the_three_immutable_files_is_missing",
509 );
510 let test_dir =
511 "error_when_one_of_the_three_immutable_files_is_missing/cardano_database";
512 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
513 .with_immutables(&[1, 2])
514 .build();
515
516 let file_to_remove = cardano_db.get_immutable_dir().join("00002.chunk");
517 std::fs::remove_file(file_to_remove).unwrap();
518
519 let db_directory = cardano_db.get_dir().to_path_buf();
520 let snapshotter = CompressedArchiveSnapshotter::new(
521 db_directory.clone(),
522 db_directory.parent().unwrap().join("snapshot_dest"),
523 CompressionAlgorithm::Gzip,
524 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
525 Arc::new(MockAncillarySigner::new()),
526 TestLogger::stdout(),
527 )
528 .unwrap();
529
530 let builder = ImmutableArtifactBuilder::new(
531 work_dir,
532 vec![Arc::new(MockImmutableFilesUploader::new())],
533 Arc::new(snapshotter),
534 TestLogger::stdout(),
535 )
536 .unwrap();
537
538 builder
539 .immutable_archives_paths_creating_the_missing_ones(2)
540 .await
541 .expect_err(
542 "Should return an error when one of the three immutable files is missing",
543 );
544 }
545
546 #[tokio::test]
547 async fn return_error_when_an_immutable_file_trio_is_missing() {
548 let work_dir =
549 get_builder_work_dir("return_error_when_an_immutable_file_trio_is_missing");
550 let test_dir = "error_when_an_immutable_file_trio_is_missing/cardano_database";
551 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
552 .with_immutables(&[1, 3])
553 .build();
554
555 let db_directory = cardano_db.get_dir().to_path_buf();
556 let snapshotter = CompressedArchiveSnapshotter::new(
557 db_directory.clone(),
558 db_directory.parent().unwrap().join("snapshot_dest"),
559 CompressionAlgorithm::Gzip,
560 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
561 Arc::new(MockAncillarySigner::new()),
562 TestLogger::stdout(),
563 )
564 .unwrap();
565
566 let builder = ImmutableArtifactBuilder::new(
567 work_dir,
568 vec![Arc::new(MockImmutableFilesUploader::new())],
569 Arc::new(snapshotter),
570 TestLogger::stdout(),
571 )
572 .unwrap();
573
574 builder
575 .immutable_archives_paths_creating_the_missing_ones(3)
576 .await
577 .expect_err("Should return an error when an immutable file trio is missing");
578 }
579
580 #[tokio::test]
581 async fn return_error_when_immutable_file_number_is_not_produced_yet() {
582 let work_dir =
583 get_builder_work_dir("return_error_when_immutable_file_number_is_not_produced_yet");
584 let test_dir = "error_when_up_to_immutable_file_number_is_missing/cardano_database";
585 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
586 .with_immutables(&[1, 2])
587 .build();
588
589 let db_directory = cardano_db.get_dir().to_path_buf();
590 let snapshotter = CompressedArchiveSnapshotter::new(
591 db_directory.clone(),
592 db_directory.parent().unwrap().join("snapshot_dest"),
593 CompressionAlgorithm::Gzip,
594 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
595 Arc::new(MockAncillarySigner::new()),
596 TestLogger::stdout(),
597 )
598 .unwrap();
599
600 let builder = ImmutableArtifactBuilder::new(
601 work_dir,
602 vec![Arc::new(MockImmutableFilesUploader::new())],
603 Arc::new(snapshotter),
604 TestLogger::stdout(),
605 )
606 .unwrap();
607
608 builder
609 .immutable_archives_paths_creating_the_missing_ones(3)
610 .await
611 .expect_err("Should return an error when an immutable file trio is missing");
612 }
613
614 #[test]
615 fn test_retrieve_existing_snapshot_archive() {
616 let work_dir = get_builder_work_dir("retrieve_existing_snapshot_archive(");
617 let file_name = "whatever.txt";
618
619 let builder = ImmutableArtifactBuilder::new(
620 work_dir.clone(),
621 vec![Arc::new(MockImmutableFilesUploader::new())],
622 Arc::new(MockSnapshotter::new()),
623 TestLogger::stdout(),
624 )
625 .unwrap();
626
627 assert_eq!(builder.retrieve_existing_snapshot_archive(file_name), None);
628
629 fs::File::create(work_dir.join(file_name)).unwrap();
630
631 assert_eq!(
632 builder.retrieve_existing_snapshot_archive(file_name),
633 Some(work_dir.join(file_name))
634 );
635 }
636
637 #[tokio::test]
638 async fn return_all_archives_but_not_rebuild_archives_already_compressed() {
639 let work_dir = get_builder_work_dir("return_all_archives_but_not_rebuild_archives");
640 let test_dir = "return_all_archives_but_not_rebuild_archives/cardano_database";
641 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
642 .with_immutables(&[1, 2, 3])
643 .build();
644
645 let db_directory = cardano_db.get_dir().to_path_buf();
646 let snapshotter = CompressedArchiveSnapshotter::new(
647 db_directory.clone(),
648 db_directory.parent().unwrap().join("snapshot_dest"),
649 CompressionAlgorithm::Gzip,
650 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
651 Arc::new(MockAncillarySigner::new()),
652 TestLogger::stdout(),
653 )
654 .unwrap();
655
656 create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
657 create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
658
659 let builder = ImmutableArtifactBuilder::new(
660 work_dir.clone(),
661 vec![Arc::new(MockImmutableFilesUploader::new())],
662 Arc::new(snapshotter),
663 TestLogger::stdout(),
664 )
665 .unwrap();
666
667 let (archive_paths, _) = builder
668 .immutable_archives_paths_creating_the_missing_ones(3)
669 .await
670 .unwrap();
671
672 assert_equivalent(
673 archive_paths,
674 vec![
675 work_dir.join("00001.tar.gz"),
676 work_dir.join("00002.tar.gz"),
677 work_dir.join("00003.tar.gz"),
678 ],
679 );
680 assert_file_content!(work_dir.join("00001.tar.gz"), "00001 content");
682 assert_file_content!(work_dir.join("00002.tar.gz"), "00002 content");
683 }
684
685 #[tokio::test]
686 async fn return_all_archives_paths_even_if_all_archives_already_exist() {
687 let work_dir =
688 get_builder_work_dir("return_all_archives_paths_even_if_all_archives_exist");
689 let mut snapshotter = MockSnapshotter::new();
690 snapshotter
691 .expect_compression_algorithm()
692 .returning(|| CompressionAlgorithm::Gzip);
693
694 create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
695 create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
696 create_fake_file(&work_dir.join("00003.tar.gz"), "00003 content");
697
698 let builder = ImmutableArtifactBuilder::new(
699 work_dir.clone(),
700 vec![Arc::new(MockImmutableFilesUploader::new())],
701 Arc::new(snapshotter),
702 TestLogger::stdout(),
703 )
704 .unwrap();
705
706 let (archive_paths, _) = builder
707 .immutable_archives_paths_creating_the_missing_ones(3)
708 .await
709 .unwrap();
710
711 assert_equivalent(
712 archive_paths,
713 vec![
714 work_dir.join("00001.tar.gz"),
715 work_dir.join("00002.tar.gz"),
716 work_dir.join("00003.tar.gz"),
717 ],
718 )
719 }
720 }
721
722 mod upload {
723 use mithril_common::test_utils::TempDir;
724
725 use super::MockImmutableFilesUploader;
726
727 use super::*;
728
729 #[test]
730 fn create_immutable_builder_should_error_when_no_uploader() {
731 let result = ImmutableArtifactBuilder::new(
732 get_builder_work_dir("create_immutable_builder_should_error_when_no_uploader"),
733 vec![],
734 Arc::new(DumbSnapshotter::default()),
735 TestLogger::stdout(),
736 );
737
738 assert!(result.is_err(), "Should return an error when no uploaders")
739 }
740
741 #[tokio::test]
742 async fn upload_immutable_archives_should_log_upload_errors() {
743 let log_path = TempDir::create(
744 "immutable",
745 "upload_immutable_archives_should_log_upload_errors",
746 )
747 .join("test.log");
748
749 let mut uploader = MockImmutableFilesUploader::new();
750 uploader
751 .expect_batch_upload()
752 .return_once(|_, _| Err(anyhow!("Failure while uploading...")));
753
754 {
755 let builder = ImmutableArtifactBuilder::new(
756 get_builder_work_dir("upload_immutable_archives_should_log_upload_errors"),
757 vec![Arc::new(uploader)],
758 Arc::new(MockSnapshotter::new()),
759 TestLogger::file(&log_path),
760 )
761 .unwrap();
762
763 let _ = builder
764 .upload_immutable_archives(
765 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
766 CompressionAlgorithm::Gzip,
767 )
768 .await;
769 }
770
771 let logs = std::fs::read_to_string(&log_path).unwrap();
772 assert!(logs.contains("Failure while uploading..."));
773 }
774
775 #[tokio::test]
776 async fn upload_immutable_archives_should_error_when_no_location_is_returned() {
777 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> =
778 vec![Arc::new(fake_uploader_returning_error())];
779
780 let builder = ImmutableArtifactBuilder::new(
781 get_builder_work_dir("upload_immutable_archives_should_error_when_no_location"),
782 uploaders,
783 Arc::new(MockSnapshotter::new()),
784 TestLogger::stdout(),
785 )
786 .unwrap();
787
788 let result = builder
789 .upload_immutable_archives(
790 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
791 CompressionAlgorithm::Gzip,
792 )
793 .await;
794
795 assert!(
796 result.is_err(),
797 "Should return an error when no location is returned"
798 );
799 }
800
801 #[tokio::test]
802 async fn upload_immutable_archives_should_return_location_even_with_uploaders_errors() {
803 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> = vec![
804 Arc::new(fake_uploader_returning_error()),
805 Arc::new(fake_uploader(
806 vec!["01.tar.gz", "02.tar.gz"],
807 "archive_2.tar.gz",
808 Some(CompressionAlgorithm::Gzip),
809 )),
810 Arc::new(fake_uploader_returning_error()),
811 ];
812
813 let builder = ImmutableArtifactBuilder::new(
814 get_builder_work_dir(
815 "upload_immutable_archives_should_return_location_even_with_uploaders_errors",
816 ),
817 uploaders,
818 Arc::new(MockSnapshotter::new()),
819 TestLogger::stdout(),
820 )
821 .unwrap();
822
823 let archive_paths = builder
824 .upload_immutable_archives(
825 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
826 CompressionAlgorithm::Gzip,
827 )
828 .await
829 .unwrap();
830
831 assert_equivalent(
832 archive_paths,
833 vec![ImmutablesLocation::CloudStorage {
834 uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
835 compression_algorithm: Some(CompressionAlgorithm::Gzip),
836 }],
837 )
838 }
839
840 #[tokio::test]
841 async fn upload_immutable_archives_should_return_all_uploaders_returned_locations() {
842 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> = vec![
843 Arc::new(fake_uploader(
844 vec!["01.tar.gz", "02.tar.gz"],
845 "archive_1.tar.gz",
846 Some(CompressionAlgorithm::Gzip),
847 )),
848 Arc::new(fake_uploader(
849 vec!["01.tar.gz", "02.tar.gz"],
850 "archive_2.tar.gz",
851 Some(CompressionAlgorithm::Gzip),
852 )),
853 ];
854
855 let builder = ImmutableArtifactBuilder::new(
856 get_builder_work_dir(
857 "upload_immutable_archives_should_return_all_uploaders_returned_locations",
858 ),
859 uploaders,
860 Arc::new(MockSnapshotter::new()),
861 TestLogger::stdout(),
862 )
863 .unwrap();
864
865 let archive_paths = builder
866 .upload_immutable_archives(
867 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
868 CompressionAlgorithm::Gzip,
869 )
870 .await
871 .unwrap();
872
873 assert_equivalent(
874 archive_paths,
875 vec![
876 ImmutablesLocation::CloudStorage {
877 uri: MultiFilesUri::Template(TemplateUri("archive_1.tar.gz".to_string())),
878 compression_algorithm: Some(CompressionAlgorithm::Gzip),
879 },
880 ImmutablesLocation::CloudStorage {
881 uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
882 compression_algorithm: Some(CompressionAlgorithm::Gzip),
883 },
884 ],
885 )
886 }
887 }
888
889 mod batch_upload {
890 use crate::file_uploaders::FileUploadRetryPolicy;
891 use crate::tools::url_sanitizer::SanitizedUrlWithTrailingSlash;
892 use mithril_common::test_utils::TempDir;
893
894 use super::*;
895
896 fn create_fake_archive(dir: &Path, name: &str) -> PathBuf {
897 let file_path = dir.join(name);
898 create_fake_file(
899 &file_path,
900 "I swear, this is an archive, not a temporary test file.",
901 );
902 file_path
903 }
904
905 #[tokio::test]
906 async fn extract_archive_name_to_deduce_template_location() {
907 let source_dir = TempDir::create(
908 "immutable",
909 "extract_archive_name_to_deduce_template_location_source",
910 );
911 let target_dir = TempDir::create(
912 "immutable",
913 "extract_archive_name_to_deduce_template_location_target",
914 );
915
916 let archive_1 = create_fake_archive(&source_dir, "00001.tar.gz");
917 let archive_2 = create_fake_archive(&source_dir, "00002.tar.gz");
918
919 let url_prefix =
920 SanitizedUrlWithTrailingSlash::parse("http://test.com:8080/base-root").unwrap();
921 let uploader = LocalUploader::new(
922 url_prefix,
923 &target_dir,
924 FileUploadRetryPolicy::never(),
925 TestLogger::stdout(),
926 );
927 let location = ImmutableFilesUploader::batch_upload(
928 &uploader,
929 &[archive_1.clone(), archive_2.clone()],
930 None,
931 )
932 .await
933 .expect("local upload should not fail");
934
935 assert!(target_dir.join(archive_1.file_name().unwrap()).exists());
936 assert!(target_dir.join(archive_2.file_name().unwrap()).exists());
937
938 let expected_location = ImmutablesLocation::CloudStorage {
939 uri: MultiFilesUri::Template(TemplateUri(
940 "http://test.com:8080/base-root/{immutable_file_number}.tar.gz".to_string(),
941 )),
942 compression_algorithm: None,
943 };
944 assert_eq!(expected_location, location);
945 }
946
947 #[tokio::test]
948 async fn returns_error_when_uploaded_filename_not_templatable_without_5_digits() {
949 let source_dir = TempDir::create(
950 "immutable",
951 "returns_error_when_uploaded_filename_not_templatable",
952 );
953 let target_dir = TempDir::create(
954 "immutable",
955 "returns_error_when_uploaded_filename_not_templatable",
956 );
957
958 let archive = create_fake_archive(&source_dir, "not-templatable.tar.gz");
959
960 let url_prefix =
961 SanitizedUrlWithTrailingSlash::parse("http://test.com:8080/base-root").unwrap();
962 let uploader = LocalUploader::new(
963 url_prefix,
964 &target_dir,
965 FileUploadRetryPolicy::never(),
966 TestLogger::stdout(),
967 );
968
969 ImmutableFilesUploader::batch_upload(&uploader, &[archive], None)
970 .await
971 .expect_err("Should return an error when not template found");
972 }
973 }
974
975 mod immutable_file_number_extractor {
976 use super::*;
977
978 #[test]
979 fn returns_none_when_not_templatable_without_5_digits() {
980 let template = immmutable_file_number_extractor("not-templatable.tar.gz").unwrap();
981
982 assert!(template.is_none());
983 }
984
985 #[test]
986 fn returns_template() {
987 let template =
988 immmutable_file_number_extractor("http://whatever/00001.tar.gz").unwrap();
989
990 assert_eq!(
991 template,
992 Some("http://whatever/{immutable_file_number}.tar.gz".to_string())
993 );
994 }
995
996 #[test]
997 fn replaces_last_occurence_of_5_digits() {
998 let template =
999 immmutable_file_number_extractor("http://00001/whatever/00001.tar.gz").unwrap();
1000
1001 assert_eq!(
1002 template,
1003 Some("http://00001/whatever/{immutable_file_number}.tar.gz".to_string())
1004 );
1005 }
1006
1007 #[test]
1008 fn replaces_last_occurence_when_more_than_5_digits() {
1009 let template =
1010 immmutable_file_number_extractor("http://whatever/123456789.tar.gz").unwrap();
1011
1012 assert_eq!(
1013 template,
1014 Some("http://whatever/1234{immutable_file_number}.tar.gz".to_string())
1015 );
1016 }
1017 }
1018
1019 #[test]
1020 fn should_compute_the_total_and_average_size_of_the_immutables() {
1021 let test_dir = "should_compute_the_average_size_of_the_immutables/cardano_database";
1022
1023 let immutable_trio_file_size = 777;
1024 let ledger_file_size = 6666;
1025 let volatile_file_size = 99;
1026
1027 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
1028 .with_immutables(&[1, 2, 3])
1029 .set_immutable_trio_file_size(immutable_trio_file_size)
1030 .with_ledger_files(&["437", "537", "637", "737"])
1031 .set_ledger_file_size(ledger_file_size)
1032 .with_volatile_files(&["blocks-0.dat", "blocks-1.dat", "blocks-2.dat"])
1033 .set_volatile_file_size(volatile_file_size)
1034 .build();
1035
1036 let (total_size, average_size) =
1037 ImmutableArtifactBuilder::compute_total_and_average_uncompressed_size(
1038 cardano_db.get_immutable_dir(),
1039 2,
1040 )
1041 .unwrap();
1042
1043 let expected_average_size = immutable_trio_file_size;
1044 let expected_total_size = immutable_trio_file_size * 2;
1045 assert_eq!(expected_total_size, total_size);
1046 assert_eq!(expected_average_size, average_size);
1047 }
1048
1049 #[test]
1050 fn should_return_an_error_when_compute_the_total_and_average_size_of_the_immutables_with_zero()
1051 {
1052 let test_dir = &format!("{}/cardano_database", current_function!());
1053
1054 let cardano_db = DummyCardanoDbBuilder::new(test_dir).build();
1055
1056 ImmutableArtifactBuilder::compute_total_and_average_uncompressed_size(
1057 cardano_db.get_immutable_dir(),
1058 0,
1059 )
1060 .expect_err("Should return an error when no immutable file number");
1061 }
1062}