1use std::{fs, path::PathBuf, sync::Arc};
2
3use anyhow::{Context, anyhow};
4use async_trait::async_trait;
5use regex::Regex;
6use slog::{Logger, error};
7
8use mithril_common::{
9 StdResult,
10 entities::{CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation, MultiFilesUri},
11 logging::LoggerExtensions,
12};
13
14use crate::{
15 DumbUploader, FileUploader,
16 file_uploaders::{CloudUploader, LocalUploader},
17 services::Snapshotter,
18};
19
20fn immutable_file_number_extractor(file_uri: &str) -> StdResult<Option<String>> {
21 let regex = Regex::new(r".*(\d{5})")?;
22
23 Ok(regex
24 .captures(file_uri)
25 .and_then(|mat| mat.get(1))
26 .map(|immutable_match| {
27 let mut template = file_uri.to_string();
28 template.replace_range(immutable_match.range(), "{immutable_file_number}");
29
30 template
31 }))
32}
33
34#[cfg_attr(test, mockall::automock)]
36#[async_trait]
37pub trait ImmutableFilesUploader: Send + Sync {
38 async fn batch_upload(
40 &self,
41 filepaths: &[PathBuf],
42 compression_algorithm: Option<CompressionAlgorithm>,
43 ) -> StdResult<ImmutablesLocation>;
44}
45
46#[derive(Debug)]
47pub struct ImmutablesUpload {
48 pub locations: Vec<ImmutablesLocation>,
49 pub average_size: u64,
50 pub total_size: u64,
51}
52
53#[async_trait]
54impl ImmutableFilesUploader for DumbUploader {
55 async fn batch_upload(
56 &self,
57 filepaths: &[PathBuf],
58 compression_algorithm: Option<CompressionAlgorithm>,
59 ) -> StdResult<ImmutablesLocation> {
60 let last_file_path = filepaths.last().ok_or_else(|| {
61 anyhow!("No file to upload with 'DumbUploader' as the filepaths list is empty")
62 })?;
63
64 let template_uri = MultiFilesUri::extract_template_from_uris(
65 vec![self.upload(last_file_path).await?.into()],
66 immutable_file_number_extractor,
67 )?
68 .ok_or_else(|| {
69 anyhow!("No matching template found in the uploaded files with 'DumbUploader'")
70 })?;
71
72 Ok(ImmutablesLocation::CloudStorage {
73 uri: MultiFilesUri::Template(template_uri),
74 compression_algorithm,
75 })
76 }
77}
78
79#[async_trait]
80impl ImmutableFilesUploader for LocalUploader {
81 async fn batch_upload(
82 &self,
83 filepaths: &[PathBuf],
84 compression_algorithm: Option<CompressionAlgorithm>,
85 ) -> StdResult<ImmutablesLocation> {
86 let mut file_uris = Vec::new();
87 for filepath in filepaths {
88 file_uris.push(self.upload(filepath).await?.into());
89 }
90
91 let template_uri =
92 MultiFilesUri::extract_template_from_uris(file_uris, immutable_file_number_extractor)?
93 .ok_or_else(|| {
94 anyhow!("No matching template found in the uploaded files with 'LocalUploader'")
95 })?;
96
97 Ok(ImmutablesLocation::CloudStorage {
98 uri: MultiFilesUri::Template(template_uri),
99 compression_algorithm,
100 })
101 }
102}
103
104#[async_trait]
105impl ImmutableFilesUploader for CloudUploader {
106 async fn batch_upload(
107 &self,
108 filepaths: &[PathBuf],
109 compression_algorithm: Option<CompressionAlgorithm>,
110 ) -> StdResult<ImmutablesLocation> {
111 let mut file_uris = Vec::new();
112 for filepath in filepaths {
113 file_uris.push(self.upload(filepath).await?.into());
114 }
115
116 let template_uri =
117 MultiFilesUri::extract_template_from_uris(file_uris, immutable_file_number_extractor)?
118 .ok_or_else(|| {
119 anyhow!("No matching template found in the uploaded files with 'CloudUploader'")
120 })?;
121
122 Ok(ImmutablesLocation::CloudStorage {
123 uri: MultiFilesUri::Template(template_uri),
124 compression_algorithm,
125 })
126 }
127}
128
129pub struct ImmutableArtifactBuilder {
130 immutables_storage_dir: PathBuf,
131 uploaders: Vec<Arc<dyn ImmutableFilesUploader>>,
132 snapshotter: Arc<dyn Snapshotter>,
133 logger: Logger,
134}
135
136impl ImmutableArtifactBuilder {
137 pub fn new(
138 immutables_storage_dir: PathBuf,
139 uploaders: Vec<Arc<dyn ImmutableFilesUploader>>,
140 snapshotter: Arc<dyn Snapshotter>,
141 logger: Logger,
142 ) -> StdResult<Self> {
143 if uploaders.is_empty() {
144 return Err(anyhow!(
145 "At least one uploader is required to create an 'ImmutableArtifactBuilder'"
146 ));
147 }
148
149 if !immutables_storage_dir.exists() {
150 fs::create_dir(&immutables_storage_dir).with_context(|| {
151 format!(
152 "Can not create immutable storage directory: '{}'",
153 immutables_storage_dir.display()
154 )
155 })?;
156 }
157
158 Ok(Self {
159 immutables_storage_dir,
160 uploaders,
161 snapshotter,
162 logger: logger.new_with_component_name::<Self>(),
163 })
164 }
165
166 pub async fn upload(
167 &self,
168 up_to_immutable_file_number: ImmutableFileNumber,
169 ) -> StdResult<ImmutablesUpload> {
170 let (archives_paths, compression_algorithm) = self
171 .immutable_archives_paths_creating_the_missing_ones(up_to_immutable_file_number)
172 .await?;
173 let locations = self
174 .upload_immutable_archives(&archives_paths, compression_algorithm)
175 .await?;
176 let total_size = self
177 .snapshotter
178 .compute_immutable_files_total_uncompressed_size(up_to_immutable_file_number)
179 .await?;
180 let average_size =
181 Self::compute_average_uncompressed_size(total_size, up_to_immutable_file_number);
182
183 Ok(ImmutablesUpload {
184 locations,
185 average_size,
186 total_size,
187 })
188 }
189
190 pub async fn immutable_archives_paths_creating_the_missing_ones(
191 &self,
192 up_to_immutable_file_number: ImmutableFileNumber,
193 ) -> StdResult<(Vec<PathBuf>, CompressionAlgorithm)> {
194 let mut archive_paths = vec![];
195 let compression_algorithm = self.snapshotter.compression_algorithm();
196 const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 0;
197
198 for immutable_file_number in FIRST_IMMUTABLE_FILE_NUMBER..=up_to_immutable_file_number {
199 let archive_name_without_extension = format!("{immutable_file_number:05}");
200 let archive_name = format!(
201 "{archive_name_without_extension}.{}",
202 compression_algorithm.tar_file_extension()
203 );
204
205 if let Some(existing_archive) = self.retrieve_existing_snapshot_archive(&archive_name) {
206 archive_paths.push(existing_archive);
207 } else {
208 let snapshot = self
209 .snapshotter
210 .snapshot_immutable_trio(immutable_file_number, &archive_name_without_extension)
211 .await?;
212
213 let target_path = self.immutables_storage_dir.join(&archive_name);
214 fs::rename(snapshot.get_file_path(), &target_path).with_context(|| {
215 format!(
216 "Can not move archive of immutable {immutable_file_number} from '{}' to '{}'",
217 snapshot.get_file_path().display(),
218 target_path.display()
219 )
220 })?;
221
222 archive_paths.push(target_path);
223 }
224 }
225
226 Ok((archive_paths, compression_algorithm))
227 }
228
229 async fn upload_immutable_archives(
230 &self,
231 archive_paths: &[PathBuf],
232 compression_algorithm: CompressionAlgorithm,
233 ) -> StdResult<Vec<ImmutablesLocation>> {
234 let mut locations = Vec::new();
235 for uploader in &self.uploaders {
236 let result = uploader
237 .batch_upload(archive_paths, Some(compression_algorithm))
238 .await;
239 match result {
240 Ok(location) => {
241 locations.push(location);
242 }
243 Err(e) => {
244 error!(
245 self.logger,
246 "Failed to upload immutable archive";
247 "error" => e.to_string()
248 );
249 }
250 }
251 }
252
253 if locations.is_empty() {
254 return Err(anyhow!(
255 "Failed to upload immutable archive with all uploaders"
256 ));
257 }
258
259 Ok(locations)
260 }
261
262 fn retrieve_existing_snapshot_archive(&self, expected_archive_name: &str) -> Option<PathBuf> {
263 let expected_archive_path = self.immutables_storage_dir.join(expected_archive_name);
264 expected_archive_path.exists().then_some(expected_archive_path)
265 }
266
267 fn compute_average_uncompressed_size(
268 total_size: u64,
269 up_to_immutable_file_number: ImmutableFileNumber,
270 ) -> u64 {
271 total_size.checked_div(up_to_immutable_file_number).unwrap_or(0)
272 }
273}
274
275#[cfg(test)]
276mod tests {
277 use std::fs::File;
278 use std::io::Write;
279 use std::path::Path;
280
281 use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder;
282 use mithril_common::{
283 entities::TemplateUri,
284 test_utils::{TempDir, assert_equivalent, equivalent_to},
285 };
286
287 use crate::services::ancillary_signer::MockAncillarySigner;
288 use crate::services::{CompressedArchiveSnapshotter, DumbSnapshotter, MockSnapshotter};
289 use crate::test_tools::TestLogger;
290 use crate::tools::file_archiver::FileArchiver;
291
292 use super::*;
293
294 fn fake_uploader(
295 archive_paths: Vec<&str>,
296 location_uri: &str,
297 compression_algorithm: Option<CompressionAlgorithm>,
298 ) -> MockImmutableFilesUploader {
299 let uri = location_uri.to_string();
300 let archive_paths: Vec<_> = archive_paths.into_iter().map(String::from).collect();
301
302 let mut uploader = MockImmutableFilesUploader::new();
303 uploader
304 .expect_batch_upload()
305 .withf(move |p, algorithm| {
306 let paths: Vec<_> = p.iter().map(|s| s.to_string_lossy().into_owned()).collect();
307
308 equivalent_to(paths, archive_paths.clone()) && algorithm == &compression_algorithm
309 })
310 .times(1)
311 .return_once(move |_, _| {
312 Ok(ImmutablesLocation::CloudStorage {
313 uri: MultiFilesUri::Template(TemplateUri(uri)),
314 compression_algorithm,
315 })
316 });
317
318 uploader
319 }
320
321 fn fake_uploader_returning_error() -> MockImmutableFilesUploader {
322 let mut uploader = MockImmutableFilesUploader::new();
323 uploader
324 .expect_batch_upload()
325 .return_once(|_, _| Err(anyhow!("Failure while uploading...")));
326
327 uploader
328 }
329
330 fn create_fake_file(path: &Path, content: &str) {
331 let mut file = File::create(path).unwrap();
332 write!(file, "{content}").unwrap();
333 }
334
335 macro_rules! assert_file_content {
336 ($path:expr, $expected_content:expr) => {
337 assert!($path.exists());
338 let content = std::fs::read_to_string(&$path).unwrap();
339 assert_eq!(content, $expected_content);
340 };
341 }
342
343 fn get_builder_work_dir<N: Into<String>>(test_name: N) -> PathBuf {
344 TempDir::create("cdb_immutable_builder", test_name)
345 }
346
347 #[tokio::test]
348 async fn upload_call_archive_creation_and_upload_to_retrieve_locations() {
349 let work_dir = get_builder_work_dir("upload_call_archive_creation_and_upload");
350 let test_dir = "upload_call_archive_creation_and_upload/cardano_database";
351 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
352 .with_immutables(&[0, 1, 2])
353 .build();
354
355 let db_directory = cardano_db.get_dir().to_path_buf();
356 let snapshotter = CompressedArchiveSnapshotter::new(
357 db_directory.clone(),
358 db_directory.parent().unwrap().join("snapshot_dest"),
359 CompressionAlgorithm::Gzip,
360 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
361 Arc::new(MockAncillarySigner::new()),
362 TestLogger::stdout(),
363 )
364 .unwrap();
365
366 let uploader = fake_uploader(
367 vec![
368 work_dir.join("00000.tar.gz").to_str().unwrap(),
369 work_dir.join("00001.tar.gz").to_str().unwrap(),
370 work_dir.join("00002.tar.gz").to_str().unwrap(),
371 ],
372 "archive.tar.gz",
373 Some(CompressionAlgorithm::Gzip),
374 );
375
376 let builder = ImmutableArtifactBuilder::new(
377 work_dir,
378 vec![Arc::new(uploader)],
379 Arc::new(snapshotter),
380 TestLogger::stdout(),
381 )
382 .unwrap();
383
384 let upload = builder.upload(2).await.unwrap();
385
386 assert_equivalent(
387 upload.locations,
388 vec![ImmutablesLocation::CloudStorage {
389 uri: MultiFilesUri::Template(TemplateUri("archive.tar.gz".to_string())),
390 compression_algorithm: Some(CompressionAlgorithm::Gzip),
391 }],
392 )
393 }
394
395 #[test]
396 fn create_immutable_builder_should_create_immutable_storage_dir_if_not_exist() {
397 let work_dir = get_builder_work_dir(
398 "create_immutable_builder_should_create_immutable_storage_dir_if_not_exist",
399 );
400 let immutable_storage_dir = work_dir.join("immutable");
401
402 assert!(!immutable_storage_dir.exists());
403
404 ImmutableArtifactBuilder::new(
405 immutable_storage_dir.clone(),
406 vec![Arc::new(DumbUploader::default())],
407 Arc::new(DumbSnapshotter::default()),
408 TestLogger::stdout(),
409 )
410 .unwrap();
411
412 assert!(immutable_storage_dir.exists());
413 }
414
415 #[test]
416 fn create_immutable_builder_should_not_create_or_remove_immutable_storage_dir_if_it_exist() {
417 let immutable_storage_dir = get_builder_work_dir(
418 "create_immutable_builder_should_not_create_or_remove_immutable_storage_dir_if_it_exist",
419 );
420 let existing_file_path = immutable_storage_dir.join("file.txt");
421 create_fake_file(&existing_file_path, "existing file content");
422
423 ImmutableArtifactBuilder::new(
424 immutable_storage_dir,
425 vec![Arc::new(DumbUploader::default())],
426 Arc::new(DumbSnapshotter::default()),
427 TestLogger::stdout(),
428 )
429 .unwrap();
430
431 assert_file_content!(existing_file_path, "existing file content");
432 }
433
434 mod create_archive {
435 use super::*;
436
437 #[tokio::test]
438 async fn snapshot_immutables_files_up_to_the_given_immutable_file_number() {
439 let work_dir = get_builder_work_dir(
440 "snapshot_immutables_files_up_to_the_given_immutable_file_number",
441 );
442 let test_dir =
443 "snapshot_immutables_files_up_to_the_given_immutable_file_number/cardano_database";
444 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
445 .with_immutables(&[0, 1, 2])
446 .build();
447
448 let db_directory = cardano_db.get_dir().to_path_buf();
449 let snapshotter = CompressedArchiveSnapshotter::new(
450 db_directory.clone(),
451 db_directory.parent().unwrap().join("snapshot_dest"),
452 CompressionAlgorithm::Gzip,
453 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
454 Arc::new(MockAncillarySigner::new()),
455 TestLogger::stdout(),
456 )
457 .unwrap();
458
459 let builder = ImmutableArtifactBuilder::new(
460 work_dir.clone(),
461 vec![Arc::new(MockImmutableFilesUploader::new())],
462 Arc::new(snapshotter),
463 TestLogger::stdout(),
464 )
465 .unwrap();
466
467 let (archive_paths, _) = builder
468 .immutable_archives_paths_creating_the_missing_ones(2)
469 .await
470 .unwrap();
471
472 assert_equivalent(
473 archive_paths,
474 vec![
475 work_dir.join("00000.tar.gz"),
476 work_dir.join("00001.tar.gz"),
477 work_dir.join("00002.tar.gz"),
478 ],
479 )
480 }
481
482 #[tokio::test]
483 async fn return_error_when_one_of_the_three_immutable_files_is_missing() {
484 let work_dir = get_builder_work_dir(
485 "return_error_when_one_of_the_three_immutable_files_is_missing",
486 );
487 let test_dir =
488 "error_when_one_of_the_three_immutable_files_is_missing/cardano_database";
489 let cardano_db = DummyCardanoDbBuilder::new(test_dir).with_immutables(&[1, 2]).build();
490
491 let file_to_remove = cardano_db.get_immutable_dir().join("00002.chunk");
492 std::fs::remove_file(file_to_remove).unwrap();
493
494 let db_directory = cardano_db.get_dir().to_path_buf();
495 let snapshotter = CompressedArchiveSnapshotter::new(
496 db_directory.clone(),
497 db_directory.parent().unwrap().join("snapshot_dest"),
498 CompressionAlgorithm::Gzip,
499 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
500 Arc::new(MockAncillarySigner::new()),
501 TestLogger::stdout(),
502 )
503 .unwrap();
504
505 let builder = ImmutableArtifactBuilder::new(
506 work_dir,
507 vec![Arc::new(MockImmutableFilesUploader::new())],
508 Arc::new(snapshotter),
509 TestLogger::stdout(),
510 )
511 .unwrap();
512
513 builder
514 .immutable_archives_paths_creating_the_missing_ones(2)
515 .await
516 .expect_err(
517 "Should return an error when one of the three immutable files is missing",
518 );
519 }
520
521 #[tokio::test]
522 async fn return_error_when_an_immutable_file_trio_is_missing() {
523 let work_dir =
524 get_builder_work_dir("return_error_when_an_immutable_file_trio_is_missing");
525 let test_dir = "error_when_an_immutable_file_trio_is_missing/cardano_database";
526 let cardano_db = DummyCardanoDbBuilder::new(test_dir).with_immutables(&[1, 3]).build();
527
528 let db_directory = cardano_db.get_dir().to_path_buf();
529 let snapshotter = CompressedArchiveSnapshotter::new(
530 db_directory.clone(),
531 db_directory.parent().unwrap().join("snapshot_dest"),
532 CompressionAlgorithm::Gzip,
533 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
534 Arc::new(MockAncillarySigner::new()),
535 TestLogger::stdout(),
536 )
537 .unwrap();
538
539 let builder = ImmutableArtifactBuilder::new(
540 work_dir,
541 vec![Arc::new(MockImmutableFilesUploader::new())],
542 Arc::new(snapshotter),
543 TestLogger::stdout(),
544 )
545 .unwrap();
546
547 builder
548 .immutable_archives_paths_creating_the_missing_ones(3)
549 .await
550 .expect_err("Should return an error when an immutable file trio is missing");
551 }
552
553 #[tokio::test]
554 async fn return_error_when_immutable_file_number_is_not_produced_yet() {
555 let work_dir =
556 get_builder_work_dir("return_error_when_immutable_file_number_is_not_produced_yet");
557 let test_dir = "error_when_up_to_immutable_file_number_is_missing/cardano_database";
558 let cardano_db = DummyCardanoDbBuilder::new(test_dir).with_immutables(&[1, 2]).build();
559
560 let db_directory = cardano_db.get_dir().to_path_buf();
561 let snapshotter = CompressedArchiveSnapshotter::new(
562 db_directory.clone(),
563 db_directory.parent().unwrap().join("snapshot_dest"),
564 CompressionAlgorithm::Gzip,
565 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
566 Arc::new(MockAncillarySigner::new()),
567 TestLogger::stdout(),
568 )
569 .unwrap();
570
571 let builder = ImmutableArtifactBuilder::new(
572 work_dir,
573 vec![Arc::new(MockImmutableFilesUploader::new())],
574 Arc::new(snapshotter),
575 TestLogger::stdout(),
576 )
577 .unwrap();
578
579 builder
580 .immutable_archives_paths_creating_the_missing_ones(3)
581 .await
582 .expect_err("Should return an error when an immutable file trio is missing");
583 }
584
585 #[test]
586 fn test_retrieve_existing_snapshot_archive() {
587 let work_dir = get_builder_work_dir("retrieve_existing_snapshot_archive(");
588 let file_name = "whatever.txt";
589
590 let builder = ImmutableArtifactBuilder::new(
591 work_dir.clone(),
592 vec![Arc::new(MockImmutableFilesUploader::new())],
593 Arc::new(MockSnapshotter::new()),
594 TestLogger::stdout(),
595 )
596 .unwrap();
597
598 assert_eq!(builder.retrieve_existing_snapshot_archive(file_name), None);
599
600 fs::File::create(work_dir.join(file_name)).unwrap();
601
602 assert_eq!(
603 builder.retrieve_existing_snapshot_archive(file_name),
604 Some(work_dir.join(file_name))
605 );
606 }
607
608 #[tokio::test]
609 async fn return_all_archives_but_not_rebuild_archives_already_compressed() {
610 let work_dir = get_builder_work_dir("return_all_archives_but_not_rebuild_archives");
611 let test_dir = "return_all_archives_but_not_rebuild_archives/cardano_database";
612 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
613 .with_immutables(&[0, 1, 2, 3])
614 .build();
615
616 let db_directory = cardano_db.get_dir().to_path_buf();
617 let snapshotter = CompressedArchiveSnapshotter::new(
618 db_directory.clone(),
619 db_directory.parent().unwrap().join("snapshot_dest"),
620 CompressionAlgorithm::Gzip,
621 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
622 Arc::new(MockAncillarySigner::new()),
623 TestLogger::stdout(),
624 )
625 .unwrap();
626
627 create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
628 create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
629 create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
630
631 let builder = ImmutableArtifactBuilder::new(
632 work_dir.clone(),
633 vec![Arc::new(MockImmutableFilesUploader::new())],
634 Arc::new(snapshotter),
635 TestLogger::stdout(),
636 )
637 .unwrap();
638
639 let (archive_paths, _) = builder
640 .immutable_archives_paths_creating_the_missing_ones(3)
641 .await
642 .unwrap();
643
644 assert_equivalent(
645 archive_paths,
646 vec![
647 work_dir.join("00000.tar.gz"),
648 work_dir.join("00001.tar.gz"),
649 work_dir.join("00002.tar.gz"),
650 work_dir.join("00003.tar.gz"),
651 ],
652 );
653 assert_file_content!(work_dir.join("00000.tar.gz"), "00000 content");
655 assert_file_content!(work_dir.join("00001.tar.gz"), "00001 content");
656 assert_file_content!(work_dir.join("00002.tar.gz"), "00002 content");
657 }
658
659 #[tokio::test]
660 async fn return_all_archives_paths_even_if_all_archives_already_exist() {
661 let work_dir =
662 get_builder_work_dir("return_all_archives_paths_even_if_all_archives_exist");
663 let mut snapshotter = MockSnapshotter::new();
664 snapshotter
665 .expect_compression_algorithm()
666 .returning(|| CompressionAlgorithm::Gzip);
667
668 create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
669 create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
670 create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
671 create_fake_file(&work_dir.join("00003.tar.gz"), "00003 content");
672
673 let builder = ImmutableArtifactBuilder::new(
674 work_dir.clone(),
675 vec![Arc::new(MockImmutableFilesUploader::new())],
676 Arc::new(snapshotter),
677 TestLogger::stdout(),
678 )
679 .unwrap();
680
681 let (archive_paths, _) = builder
682 .immutable_archives_paths_creating_the_missing_ones(3)
683 .await
684 .unwrap();
685
686 assert_equivalent(
687 archive_paths,
688 vec![
689 work_dir.join("00000.tar.gz"),
690 work_dir.join("00001.tar.gz"),
691 work_dir.join("00002.tar.gz"),
692 work_dir.join("00003.tar.gz"),
693 ],
694 )
695 }
696 }
697
698 mod upload {
699 use super::MockImmutableFilesUploader;
700
701 use super::*;
702
703 #[test]
704 fn create_immutable_builder_should_error_when_no_uploader() {
705 let result = ImmutableArtifactBuilder::new(
706 get_builder_work_dir("create_immutable_builder_should_error_when_no_uploader"),
707 vec![],
708 Arc::new(DumbSnapshotter::default()),
709 TestLogger::stdout(),
710 );
711
712 assert!(result.is_err(), "Should return an error when no uploaders")
713 }
714
715 #[tokio::test]
716 async fn upload_immutable_archives_should_log_upload_errors() {
717 let (logger, log_inspector) = TestLogger::memory();
718 let mut uploader = MockImmutableFilesUploader::new();
719 uploader
720 .expect_batch_upload()
721 .return_once(|_, _| Err(anyhow!("Failure while uploading...")));
722
723 let builder = ImmutableArtifactBuilder::new(
724 get_builder_work_dir("upload_immutable_archives_should_log_upload_errors"),
725 vec![Arc::new(uploader)],
726 Arc::new(MockSnapshotter::new()),
727 logger,
728 )
729 .unwrap();
730
731 let _ = builder
732 .upload_immutable_archives(
733 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
734 CompressionAlgorithm::Gzip,
735 )
736 .await;
737
738 assert!(log_inspector.contains_log("Failure while uploading..."));
739 }
740
741 #[tokio::test]
742 async fn upload_immutable_archives_should_error_when_no_location_is_returned() {
743 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> =
744 vec![Arc::new(fake_uploader_returning_error())];
745
746 let builder = ImmutableArtifactBuilder::new(
747 get_builder_work_dir("upload_immutable_archives_should_error_when_no_location"),
748 uploaders,
749 Arc::new(MockSnapshotter::new()),
750 TestLogger::stdout(),
751 )
752 .unwrap();
753
754 let result = builder
755 .upload_immutable_archives(
756 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
757 CompressionAlgorithm::Gzip,
758 )
759 .await;
760
761 assert!(
762 result.is_err(),
763 "Should return an error when no location is returned"
764 );
765 }
766
767 #[tokio::test]
768 async fn upload_immutable_archives_should_return_location_even_with_uploaders_errors() {
769 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> = vec![
770 Arc::new(fake_uploader_returning_error()),
771 Arc::new(fake_uploader(
772 vec!["01.tar.gz", "02.tar.gz"],
773 "archive_2.tar.gz",
774 Some(CompressionAlgorithm::Gzip),
775 )),
776 Arc::new(fake_uploader_returning_error()),
777 ];
778
779 let builder = ImmutableArtifactBuilder::new(
780 get_builder_work_dir(
781 "upload_immutable_archives_should_return_location_even_with_uploaders_errors",
782 ),
783 uploaders,
784 Arc::new(MockSnapshotter::new()),
785 TestLogger::stdout(),
786 )
787 .unwrap();
788
789 let archive_paths = builder
790 .upload_immutable_archives(
791 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
792 CompressionAlgorithm::Gzip,
793 )
794 .await
795 .unwrap();
796
797 assert_equivalent(
798 archive_paths,
799 vec![ImmutablesLocation::CloudStorage {
800 uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
801 compression_algorithm: Some(CompressionAlgorithm::Gzip),
802 }],
803 )
804 }
805
806 #[tokio::test]
807 async fn upload_immutable_archives_should_return_all_uploaders_returned_locations() {
808 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> = vec![
809 Arc::new(fake_uploader(
810 vec!["01.tar.gz", "02.tar.gz"],
811 "archive_1.tar.gz",
812 Some(CompressionAlgorithm::Gzip),
813 )),
814 Arc::new(fake_uploader(
815 vec!["01.tar.gz", "02.tar.gz"],
816 "archive_2.tar.gz",
817 Some(CompressionAlgorithm::Gzip),
818 )),
819 ];
820
821 let builder = ImmutableArtifactBuilder::new(
822 get_builder_work_dir(
823 "upload_immutable_archives_should_return_all_uploaders_returned_locations",
824 ),
825 uploaders,
826 Arc::new(MockSnapshotter::new()),
827 TestLogger::stdout(),
828 )
829 .unwrap();
830
831 let archive_paths = builder
832 .upload_immutable_archives(
833 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
834 CompressionAlgorithm::Gzip,
835 )
836 .await
837 .unwrap();
838
839 assert_equivalent(
840 archive_paths,
841 vec![
842 ImmutablesLocation::CloudStorage {
843 uri: MultiFilesUri::Template(TemplateUri("archive_1.tar.gz".to_string())),
844 compression_algorithm: Some(CompressionAlgorithm::Gzip),
845 },
846 ImmutablesLocation::CloudStorage {
847 uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
848 compression_algorithm: Some(CompressionAlgorithm::Gzip),
849 },
850 ],
851 )
852 }
853 }
854
855 mod batch_upload {
856 use mithril_common::test_utils::TempDir;
857
858 use crate::file_uploaders::FileUploadRetryPolicy;
859 use crate::tools::url_sanitizer::SanitizedUrlWithTrailingSlash;
860
861 use super::*;
862
863 fn create_fake_archive(dir: &Path, name: &str) -> PathBuf {
864 let file_path = dir.join(name);
865 create_fake_file(
866 &file_path,
867 "I swear, this is an archive, not a temporary test file.",
868 );
869 file_path
870 }
871
872 #[tokio::test]
873 async fn extract_archive_name_to_deduce_template_location() {
874 let source_dir = TempDir::create(
875 "immutable",
876 "extract_archive_name_to_deduce_template_location_source",
877 );
878 let target_dir = TempDir::create(
879 "immutable",
880 "extract_archive_name_to_deduce_template_location_target",
881 );
882
883 let archive_1 = create_fake_archive(&source_dir, "00001.tar.gz");
884 let archive_2 = create_fake_archive(&source_dir, "00002.tar.gz");
885
886 let url_prefix =
887 SanitizedUrlWithTrailingSlash::parse("http://test.com:8080/base-root").unwrap();
888 let uploader = LocalUploader::new(
889 url_prefix,
890 &target_dir,
891 FileUploadRetryPolicy::never(),
892 TestLogger::stdout(),
893 );
894 let location = ImmutableFilesUploader::batch_upload(
895 &uploader,
896 &[archive_1.clone(), archive_2.clone()],
897 None,
898 )
899 .await
900 .expect("local upload should not fail");
901
902 assert!(target_dir.join(archive_1.file_name().unwrap()).exists());
903 assert!(target_dir.join(archive_2.file_name().unwrap()).exists());
904
905 let expected_location = ImmutablesLocation::CloudStorage {
906 uri: MultiFilesUri::Template(TemplateUri(
907 "http://test.com:8080/base-root/{immutable_file_number}.tar.gz".to_string(),
908 )),
909 compression_algorithm: None,
910 };
911 assert_eq!(expected_location, location);
912 }
913
914 #[tokio::test]
915 async fn returns_error_when_uploaded_filename_not_templatable_without_5_digits() {
916 let source_dir = TempDir::create(
917 "immutable",
918 "returns_error_when_uploaded_filename_not_templatable",
919 );
920 let target_dir = TempDir::create(
921 "immutable",
922 "returns_error_when_uploaded_filename_not_templatable",
923 );
924
925 let archive = create_fake_archive(&source_dir, "not-templatable.tar.gz");
926
927 let url_prefix =
928 SanitizedUrlWithTrailingSlash::parse("http://test.com:8080/base-root").unwrap();
929 let uploader = LocalUploader::new(
930 url_prefix,
931 &target_dir,
932 FileUploadRetryPolicy::never(),
933 TestLogger::stdout(),
934 );
935
936 ImmutableFilesUploader::batch_upload(&uploader, &[archive], None)
937 .await
938 .expect_err("Should return an error when not template found");
939 }
940 }
941
942 mod immutable_file_number_extractor {
943 use super::*;
944
945 #[test]
946 fn returns_none_when_not_templatable_without_5_digits() {
947 let template = immutable_file_number_extractor("not-templatable.tar.gz").unwrap();
948
949 assert!(template.is_none());
950 }
951
952 #[test]
953 fn returns_template() {
954 let template = immutable_file_number_extractor("http://whatever/00001.tar.gz").unwrap();
955
956 assert_eq!(
957 template,
958 Some("http://whatever/{immutable_file_number}.tar.gz".to_string())
959 );
960 }
961
962 #[test]
963 fn replaces_last_occurence_of_5_digits() {
964 let template =
965 immutable_file_number_extractor("http://00001/whatever/00001.tar.gz").unwrap();
966
967 assert_eq!(
968 template,
969 Some("http://00001/whatever/{immutable_file_number}.tar.gz".to_string())
970 );
971 }
972
973 #[test]
974 fn replaces_last_occurence_when_more_than_5_digits() {
975 let template =
976 immutable_file_number_extractor("http://whatever/123456789.tar.gz").unwrap();
977
978 assert_eq!(
979 template,
980 Some("http://whatever/1234{immutable_file_number}.tar.gz".to_string())
981 );
982 }
983 }
984
985 #[test]
986 fn compute_average_uncompressed_size() {
987 let total_size = 14;
988 let average_size_up_to_immutable_2 =
989 ImmutableArtifactBuilder::compute_average_uncompressed_size(total_size, 2);
990 assert_eq!(7, average_size_up_to_immutable_2);
991
992 let average_size_up_to_immutable_5 =
994 ImmutableArtifactBuilder::compute_average_uncompressed_size(total_size, 5);
995 assert_eq!(2, average_size_up_to_immutable_5);
996
997 let average_size_up_to_immutable_0 =
999 ImmutableArtifactBuilder::compute_average_uncompressed_size(total_size, 0);
1000 assert_eq!(0, average_size_up_to_immutable_0);
1001 }
1002}