1use std::{fs, path::PathBuf, sync::Arc};
2
3use anyhow::{Context, anyhow};
4use async_trait::async_trait;
5use regex::Regex;
6use slog::{Logger, error};
7
8use mithril_common::{
9 StdResult,
10 entities::{CompressionAlgorithm, ImmutableFileNumber, ImmutablesLocation, MultiFilesUri},
11 logging::LoggerExtensions,
12};
13
14use crate::{
15 DumbUploader, FileUploader,
16 file_uploaders::{CloudUploader, LocalUploader},
17 services::Snapshotter,
18};
19
20fn immutable_file_number_extractor(file_uri: &str) -> StdResult<Option<String>> {
21 let regex = Regex::new(r".*(\d{5})")?;
22
23 Ok(regex
24 .captures(file_uri)
25 .and_then(|mat| mat.get(1))
26 .map(|immutable_match| {
27 let mut template = file_uri.to_string();
28 template.replace_range(immutable_match.range(), "{immutable_file_number}");
29
30 template
31 }))
32}
33
34#[cfg_attr(test, mockall::automock)]
36#[async_trait]
37pub trait ImmutableFilesUploader: Send + Sync {
38 async fn batch_upload(
40 &self,
41 filepaths: &[PathBuf],
42 compression_algorithm: Option<CompressionAlgorithm>,
43 ) -> StdResult<ImmutablesLocation>;
44}
45
46#[derive(Debug)]
47pub struct ImmutablesUpload {
48 pub locations: Vec<ImmutablesLocation>,
49 pub average_size: u64,
50 pub total_size: u64,
51}
52
53#[async_trait]
54impl ImmutableFilesUploader for DumbUploader {
55 async fn batch_upload(
56 &self,
57 filepaths: &[PathBuf],
58 compression_algorithm: Option<CompressionAlgorithm>,
59 ) -> StdResult<ImmutablesLocation> {
60 let last_file_path = filepaths.last().with_context(
61 || "No file to upload with 'DumbUploader' as the filepaths list is empty",
62 )?;
63
64 let template_uri = MultiFilesUri::extract_template_from_uris(
65 vec![self.upload(last_file_path).await?.into()],
66 immutable_file_number_extractor,
67 )?
68 .with_context(|| "No matching template found in the uploaded files with 'DumbUploader'")?;
69
70 Ok(ImmutablesLocation::CloudStorage {
71 uri: MultiFilesUri::Template(template_uri),
72 compression_algorithm,
73 })
74 }
75}
76
77#[async_trait]
78impl ImmutableFilesUploader for LocalUploader {
79 async fn batch_upload(
80 &self,
81 filepaths: &[PathBuf],
82 compression_algorithm: Option<CompressionAlgorithm>,
83 ) -> StdResult<ImmutablesLocation> {
84 let mut file_uris = Vec::new();
85 for filepath in filepaths {
86 file_uris.push(self.upload(filepath).await?.into());
87 }
88
89 let template_uri =
90 MultiFilesUri::extract_template_from_uris(file_uris, immutable_file_number_extractor)?
91 .with_context(
92 || "No matching template found in the uploaded files with 'LocalUploader'",
93 )?;
94
95 Ok(ImmutablesLocation::CloudStorage {
96 uri: MultiFilesUri::Template(template_uri),
97 compression_algorithm,
98 })
99 }
100}
101
102#[async_trait]
103impl ImmutableFilesUploader for CloudUploader {
104 async fn batch_upload(
105 &self,
106 filepaths: &[PathBuf],
107 compression_algorithm: Option<CompressionAlgorithm>,
108 ) -> StdResult<ImmutablesLocation> {
109 let mut file_uris = Vec::new();
110 for filepath in filepaths {
111 file_uris.push(self.upload(filepath).await?.into());
112 }
113
114 let template_uri =
115 MultiFilesUri::extract_template_from_uris(file_uris, immutable_file_number_extractor)?
116 .with_context(
117 || "No matching template found in the uploaded files with 'CloudUploader'",
118 )?;
119
120 Ok(ImmutablesLocation::CloudStorage {
121 uri: MultiFilesUri::Template(template_uri),
122 compression_algorithm,
123 })
124 }
125}
126
127pub struct ImmutableArtifactBuilder {
128 immutables_storage_dir: PathBuf,
129 uploaders: Vec<Arc<dyn ImmutableFilesUploader>>,
130 snapshotter: Arc<dyn Snapshotter>,
131 logger: Logger,
132}
133
134impl ImmutableArtifactBuilder {
135 pub fn new(
136 immutables_storage_dir: PathBuf,
137 uploaders: Vec<Arc<dyn ImmutableFilesUploader>>,
138 snapshotter: Arc<dyn Snapshotter>,
139 logger: Logger,
140 ) -> StdResult<Self> {
141 if uploaders.is_empty() {
142 return Err(anyhow!(
143 "At least one uploader is required to create an 'ImmutableArtifactBuilder'"
144 ));
145 }
146
147 if !immutables_storage_dir.exists() {
148 fs::create_dir(&immutables_storage_dir).with_context(|| {
149 format!(
150 "Can not create immutable storage directory: '{}'",
151 immutables_storage_dir.display()
152 )
153 })?;
154 }
155
156 Ok(Self {
157 immutables_storage_dir,
158 uploaders,
159 snapshotter,
160 logger: logger.new_with_component_name::<Self>(),
161 })
162 }
163
164 pub async fn upload(
165 &self,
166 up_to_immutable_file_number: ImmutableFileNumber,
167 ) -> StdResult<ImmutablesUpload> {
168 let (archives_paths, compression_algorithm) = self
169 .immutable_archives_paths_creating_the_missing_ones(up_to_immutable_file_number)
170 .await?;
171 let locations = self
172 .upload_immutable_archives(&archives_paths, compression_algorithm)
173 .await?;
174 let total_size = self
175 .snapshotter
176 .compute_immutable_files_total_uncompressed_size(up_to_immutable_file_number)
177 .await?;
178 let average_size =
179 Self::compute_average_uncompressed_size(total_size, up_to_immutable_file_number);
180
181 Ok(ImmutablesUpload {
182 locations,
183 average_size,
184 total_size,
185 })
186 }
187
188 pub async fn immutable_archives_paths_creating_the_missing_ones(
189 &self,
190 up_to_immutable_file_number: ImmutableFileNumber,
191 ) -> StdResult<(Vec<PathBuf>, CompressionAlgorithm)> {
192 let mut archive_paths = vec![];
193 let compression_algorithm = self.snapshotter.compression_algorithm();
194 const FIRST_IMMUTABLE_FILE_NUMBER: ImmutableFileNumber = 0;
195
196 for immutable_file_number in FIRST_IMMUTABLE_FILE_NUMBER..=up_to_immutable_file_number {
197 let archive_name_without_extension = format!("{immutable_file_number:05}");
198 let archive_name = format!(
199 "{archive_name_without_extension}.{}",
200 compression_algorithm.tar_file_extension()
201 );
202
203 if let Some(existing_archive) = self.retrieve_existing_snapshot_archive(&archive_name) {
204 archive_paths.push(existing_archive);
205 } else {
206 let snapshot = self
207 .snapshotter
208 .snapshot_immutable_trio(immutable_file_number, &archive_name_without_extension)
209 .await?;
210
211 let target_path = self.immutables_storage_dir.join(&archive_name);
212 fs::rename(snapshot.get_file_path(), &target_path).with_context(|| {
213 format!(
214 "Can not move archive of immutable {immutable_file_number} from '{}' to '{}'",
215 snapshot.get_file_path().display(),
216 target_path.display()
217 )
218 })?;
219
220 archive_paths.push(target_path);
221 }
222 }
223
224 Ok((archive_paths, compression_algorithm))
225 }
226
227 async fn upload_immutable_archives(
228 &self,
229 archive_paths: &[PathBuf],
230 compression_algorithm: CompressionAlgorithm,
231 ) -> StdResult<Vec<ImmutablesLocation>> {
232 let mut locations = Vec::new();
233 for uploader in &self.uploaders {
234 let result = uploader
235 .batch_upload(archive_paths, Some(compression_algorithm))
236 .await;
237 match result {
238 Ok(location) => {
239 locations.push(location);
240 }
241 Err(e) => {
242 error!(
243 self.logger,
244 "Failed to upload immutable archive";
245 "error" => e.to_string()
246 );
247 }
248 }
249 }
250
251 if locations.is_empty() {
252 return Err(anyhow!(
253 "Failed to upload immutable archive with all uploaders"
254 ));
255 }
256
257 Ok(locations)
258 }
259
260 fn retrieve_existing_snapshot_archive(&self, expected_archive_name: &str) -> Option<PathBuf> {
261 let expected_archive_path = self.immutables_storage_dir.join(expected_archive_name);
262 expected_archive_path.exists().then_some(expected_archive_path)
263 }
264
265 fn compute_average_uncompressed_size(
266 total_size: u64,
267 up_to_immutable_file_number: ImmutableFileNumber,
268 ) -> u64 {
269 total_size.checked_div(up_to_immutable_file_number).unwrap_or(0)
270 }
271}
272
273#[cfg(test)]
274mod tests {
275 use std::fs::File;
276 use std::io::Write;
277 use std::path::Path;
278
279 use mithril_cardano_node_internal_database::test::DummyCardanoDbBuilder;
280 use mithril_common::{
281 entities::TemplateUri,
282 test::{TempDir, assert_equivalent, equivalent_to},
283 };
284
285 use crate::services::ancillary_signer::MockAncillarySigner;
286 use crate::services::{CompressedArchiveSnapshotter, DumbSnapshotter, MockSnapshotter};
287 use crate::test::TestLogger;
288 use crate::tools::file_archiver::FileArchiver;
289
290 use super::*;
291
292 fn fake_uploader(
293 archive_paths: Vec<&str>,
294 location_uri: &str,
295 compression_algorithm: Option<CompressionAlgorithm>,
296 ) -> MockImmutableFilesUploader {
297 let uri = location_uri.to_string();
298 let archive_paths: Vec<_> = archive_paths.into_iter().map(String::from).collect();
299
300 let mut uploader = MockImmutableFilesUploader::new();
301 uploader
302 .expect_batch_upload()
303 .withf(move |p, algorithm| {
304 let paths: Vec<_> = p.iter().map(|s| s.to_string_lossy().into_owned()).collect();
305
306 equivalent_to(paths, archive_paths.clone()) && algorithm == &compression_algorithm
307 })
308 .times(1)
309 .return_once(move |_, _| {
310 Ok(ImmutablesLocation::CloudStorage {
311 uri: MultiFilesUri::Template(TemplateUri(uri)),
312 compression_algorithm,
313 })
314 });
315
316 uploader
317 }
318
319 fn fake_uploader_returning_error() -> MockImmutableFilesUploader {
320 let mut uploader = MockImmutableFilesUploader::new();
321 uploader
322 .expect_batch_upload()
323 .return_once(|_, _| Err(anyhow!("Failure while uploading...")));
324
325 uploader
326 }
327
328 fn create_fake_file(path: &Path, content: &str) {
329 let mut file = File::create(path).unwrap();
330 write!(file, "{content}").unwrap();
331 }
332
333 macro_rules! assert_file_content {
334 ($path:expr, $expected_content:expr) => {
335 assert!($path.exists());
336 let content = std::fs::read_to_string(&$path).unwrap();
337 assert_eq!(content, $expected_content);
338 };
339 }
340
341 fn get_builder_work_dir<N: Into<String>>(test_name: N) -> PathBuf {
342 TempDir::create("cdb_immutable_builder", test_name)
343 }
344
345 #[tokio::test]
346 async fn upload_call_archive_creation_and_upload_to_retrieve_locations() {
347 let work_dir = get_builder_work_dir("upload_call_archive_creation_and_upload");
348 let test_dir = "upload_call_archive_creation_and_upload/cardano_database";
349 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
350 .with_immutables(&[0, 1, 2])
351 .build();
352
353 let db_directory = cardano_db.get_dir().to_path_buf();
354 let snapshotter = CompressedArchiveSnapshotter::new(
355 db_directory.clone(),
356 db_directory.parent().unwrap().join("snapshot_dest"),
357 CompressionAlgorithm::Gzip,
358 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
359 Arc::new(MockAncillarySigner::new()),
360 TestLogger::stdout(),
361 )
362 .unwrap();
363
364 let uploader = fake_uploader(
365 vec![
366 work_dir.join("00000.tar.gz").to_str().unwrap(),
367 work_dir.join("00001.tar.gz").to_str().unwrap(),
368 work_dir.join("00002.tar.gz").to_str().unwrap(),
369 ],
370 "archive.tar.gz",
371 Some(CompressionAlgorithm::Gzip),
372 );
373
374 let builder = ImmutableArtifactBuilder::new(
375 work_dir,
376 vec![Arc::new(uploader)],
377 Arc::new(snapshotter),
378 TestLogger::stdout(),
379 )
380 .unwrap();
381
382 let upload = builder.upload(2).await.unwrap();
383
384 assert_equivalent!(
385 upload.locations,
386 vec![ImmutablesLocation::CloudStorage {
387 uri: MultiFilesUri::Template(TemplateUri("archive.tar.gz".to_string())),
388 compression_algorithm: Some(CompressionAlgorithm::Gzip),
389 }],
390 )
391 }
392
393 #[test]
394 fn create_immutable_builder_should_create_immutable_storage_dir_if_not_exist() {
395 let work_dir = get_builder_work_dir(
396 "create_immutable_builder_should_create_immutable_storage_dir_if_not_exist",
397 );
398 let immutable_storage_dir = work_dir.join("immutable");
399
400 assert!(!immutable_storage_dir.exists());
401
402 ImmutableArtifactBuilder::new(
403 immutable_storage_dir.clone(),
404 vec![Arc::new(DumbUploader::default())],
405 Arc::new(DumbSnapshotter::default()),
406 TestLogger::stdout(),
407 )
408 .unwrap();
409
410 assert!(immutable_storage_dir.exists());
411 }
412
413 #[test]
414 fn create_immutable_builder_should_not_create_or_remove_immutable_storage_dir_if_it_exist() {
415 let immutable_storage_dir = get_builder_work_dir(
416 "create_immutable_builder_should_not_create_or_remove_immutable_storage_dir_if_it_exist",
417 );
418 let existing_file_path = immutable_storage_dir.join("file.txt");
419 create_fake_file(&existing_file_path, "existing file content");
420
421 ImmutableArtifactBuilder::new(
422 immutable_storage_dir,
423 vec![Arc::new(DumbUploader::default())],
424 Arc::new(DumbSnapshotter::default()),
425 TestLogger::stdout(),
426 )
427 .unwrap();
428
429 assert_file_content!(existing_file_path, "existing file content");
430 }
431
432 mod create_archive {
433 use super::*;
434
435 #[tokio::test]
436 async fn snapshot_immutables_files_up_to_the_given_immutable_file_number() {
437 let work_dir = get_builder_work_dir(
438 "snapshot_immutables_files_up_to_the_given_immutable_file_number",
439 );
440 let test_dir =
441 "snapshot_immutables_files_up_to_the_given_immutable_file_number/cardano_database";
442 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
443 .with_immutables(&[0, 1, 2])
444 .build();
445
446 let db_directory = cardano_db.get_dir().to_path_buf();
447 let snapshotter = CompressedArchiveSnapshotter::new(
448 db_directory.clone(),
449 db_directory.parent().unwrap().join("snapshot_dest"),
450 CompressionAlgorithm::Gzip,
451 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
452 Arc::new(MockAncillarySigner::new()),
453 TestLogger::stdout(),
454 )
455 .unwrap();
456
457 let builder = ImmutableArtifactBuilder::new(
458 work_dir.clone(),
459 vec![Arc::new(MockImmutableFilesUploader::new())],
460 Arc::new(snapshotter),
461 TestLogger::stdout(),
462 )
463 .unwrap();
464
465 let (archive_paths, _) = builder
466 .immutable_archives_paths_creating_the_missing_ones(2)
467 .await
468 .unwrap();
469
470 assert_equivalent!(
471 archive_paths,
472 vec![
473 work_dir.join("00000.tar.gz"),
474 work_dir.join("00001.tar.gz"),
475 work_dir.join("00002.tar.gz"),
476 ],
477 )
478 }
479
480 #[tokio::test]
481 async fn return_error_when_one_of_the_three_immutable_files_is_missing() {
482 let work_dir = get_builder_work_dir(
483 "return_error_when_one_of_the_three_immutable_files_is_missing",
484 );
485 let test_dir =
486 "error_when_one_of_the_three_immutable_files_is_missing/cardano_database";
487 let cardano_db = DummyCardanoDbBuilder::new(test_dir).with_immutables(&[1, 2]).build();
488
489 let file_to_remove = cardano_db.get_immutable_dir().join("00002.chunk");
490 std::fs::remove_file(file_to_remove).unwrap();
491
492 let db_directory = cardano_db.get_dir().to_path_buf();
493 let snapshotter = CompressedArchiveSnapshotter::new(
494 db_directory.clone(),
495 db_directory.parent().unwrap().join("snapshot_dest"),
496 CompressionAlgorithm::Gzip,
497 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
498 Arc::new(MockAncillarySigner::new()),
499 TestLogger::stdout(),
500 )
501 .unwrap();
502
503 let builder = ImmutableArtifactBuilder::new(
504 work_dir,
505 vec![Arc::new(MockImmutableFilesUploader::new())],
506 Arc::new(snapshotter),
507 TestLogger::stdout(),
508 )
509 .unwrap();
510
511 builder
512 .immutable_archives_paths_creating_the_missing_ones(2)
513 .await
514 .expect_err(
515 "Should return an error when one of the three immutable files is missing",
516 );
517 }
518
519 #[tokio::test]
520 async fn return_error_when_an_immutable_file_trio_is_missing() {
521 let work_dir =
522 get_builder_work_dir("return_error_when_an_immutable_file_trio_is_missing");
523 let test_dir = "error_when_an_immutable_file_trio_is_missing/cardano_database";
524 let cardano_db = DummyCardanoDbBuilder::new(test_dir).with_immutables(&[1, 3]).build();
525
526 let db_directory = cardano_db.get_dir().to_path_buf();
527 let snapshotter = CompressedArchiveSnapshotter::new(
528 db_directory.clone(),
529 db_directory.parent().unwrap().join("snapshot_dest"),
530 CompressionAlgorithm::Gzip,
531 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
532 Arc::new(MockAncillarySigner::new()),
533 TestLogger::stdout(),
534 )
535 .unwrap();
536
537 let builder = ImmutableArtifactBuilder::new(
538 work_dir,
539 vec![Arc::new(MockImmutableFilesUploader::new())],
540 Arc::new(snapshotter),
541 TestLogger::stdout(),
542 )
543 .unwrap();
544
545 builder
546 .immutable_archives_paths_creating_the_missing_ones(3)
547 .await
548 .expect_err("Should return an error when an immutable file trio is missing");
549 }
550
551 #[tokio::test]
552 async fn return_error_when_immutable_file_number_is_not_produced_yet() {
553 let work_dir =
554 get_builder_work_dir("return_error_when_immutable_file_number_is_not_produced_yet");
555 let test_dir = "error_when_up_to_immutable_file_number_is_missing/cardano_database";
556 let cardano_db = DummyCardanoDbBuilder::new(test_dir).with_immutables(&[1, 2]).build();
557
558 let db_directory = cardano_db.get_dir().to_path_buf();
559 let snapshotter = CompressedArchiveSnapshotter::new(
560 db_directory.clone(),
561 db_directory.parent().unwrap().join("snapshot_dest"),
562 CompressionAlgorithm::Gzip,
563 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
564 Arc::new(MockAncillarySigner::new()),
565 TestLogger::stdout(),
566 )
567 .unwrap();
568
569 let builder = ImmutableArtifactBuilder::new(
570 work_dir,
571 vec![Arc::new(MockImmutableFilesUploader::new())],
572 Arc::new(snapshotter),
573 TestLogger::stdout(),
574 )
575 .unwrap();
576
577 builder
578 .immutable_archives_paths_creating_the_missing_ones(3)
579 .await
580 .expect_err("Should return an error when an immutable file trio is missing");
581 }
582
583 #[test]
584 fn test_retrieve_existing_snapshot_archive() {
585 let work_dir = get_builder_work_dir("retrieve_existing_snapshot_archive(");
586 let file_name = "whatever.txt";
587
588 let builder = ImmutableArtifactBuilder::new(
589 work_dir.clone(),
590 vec![Arc::new(MockImmutableFilesUploader::new())],
591 Arc::new(MockSnapshotter::new()),
592 TestLogger::stdout(),
593 )
594 .unwrap();
595
596 assert_eq!(builder.retrieve_existing_snapshot_archive(file_name), None);
597
598 fs::File::create(work_dir.join(file_name)).unwrap();
599
600 assert_eq!(
601 builder.retrieve_existing_snapshot_archive(file_name),
602 Some(work_dir.join(file_name))
603 );
604 }
605
606 #[tokio::test]
607 async fn return_all_archives_but_not_rebuild_archives_already_compressed() {
608 let work_dir = get_builder_work_dir("return_all_archives_but_not_rebuild_archives");
609 let test_dir = "return_all_archives_but_not_rebuild_archives/cardano_database";
610 let cardano_db = DummyCardanoDbBuilder::new(test_dir)
611 .with_immutables(&[0, 1, 2, 3])
612 .build();
613
614 let db_directory = cardano_db.get_dir().to_path_buf();
615 let snapshotter = CompressedArchiveSnapshotter::new(
616 db_directory.clone(),
617 db_directory.parent().unwrap().join("snapshot_dest"),
618 CompressionAlgorithm::Gzip,
619 Arc::new(FileArchiver::new_for_test(work_dir.join("verification"))),
620 Arc::new(MockAncillarySigner::new()),
621 TestLogger::stdout(),
622 )
623 .unwrap();
624
625 create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
626 create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
627 create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
628
629 let builder = ImmutableArtifactBuilder::new(
630 work_dir.clone(),
631 vec![Arc::new(MockImmutableFilesUploader::new())],
632 Arc::new(snapshotter),
633 TestLogger::stdout(),
634 )
635 .unwrap();
636
637 let (archive_paths, _) = builder
638 .immutable_archives_paths_creating_the_missing_ones(3)
639 .await
640 .unwrap();
641
642 assert_equivalent!(
643 archive_paths,
644 vec![
645 work_dir.join("00000.tar.gz"),
646 work_dir.join("00001.tar.gz"),
647 work_dir.join("00002.tar.gz"),
648 work_dir.join("00003.tar.gz"),
649 ],
650 );
651 assert_file_content!(work_dir.join("00000.tar.gz"), "00000 content");
653 assert_file_content!(work_dir.join("00001.tar.gz"), "00001 content");
654 assert_file_content!(work_dir.join("00002.tar.gz"), "00002 content");
655 }
656
657 #[tokio::test]
658 async fn return_all_archives_paths_even_if_all_archives_already_exist() {
659 let work_dir =
660 get_builder_work_dir("return_all_archives_paths_even_if_all_archives_exist");
661 let mut snapshotter = MockSnapshotter::new();
662 snapshotter
663 .expect_compression_algorithm()
664 .returning(|| CompressionAlgorithm::Gzip);
665
666 create_fake_file(&work_dir.join("00000.tar.gz"), "00000 content");
667 create_fake_file(&work_dir.join("00001.tar.gz"), "00001 content");
668 create_fake_file(&work_dir.join("00002.tar.gz"), "00002 content");
669 create_fake_file(&work_dir.join("00003.tar.gz"), "00003 content");
670
671 let builder = ImmutableArtifactBuilder::new(
672 work_dir.clone(),
673 vec![Arc::new(MockImmutableFilesUploader::new())],
674 Arc::new(snapshotter),
675 TestLogger::stdout(),
676 )
677 .unwrap();
678
679 let (archive_paths, _) = builder
680 .immutable_archives_paths_creating_the_missing_ones(3)
681 .await
682 .unwrap();
683
684 assert_equivalent!(
685 archive_paths,
686 vec![
687 work_dir.join("00000.tar.gz"),
688 work_dir.join("00001.tar.gz"),
689 work_dir.join("00002.tar.gz"),
690 work_dir.join("00003.tar.gz"),
691 ],
692 )
693 }
694 }
695
696 mod upload {
697 use super::MockImmutableFilesUploader;
698
699 use super::*;
700
701 #[test]
702 fn create_immutable_builder_should_error_when_no_uploader() {
703 let result = ImmutableArtifactBuilder::new(
704 get_builder_work_dir("create_immutable_builder_should_error_when_no_uploader"),
705 vec![],
706 Arc::new(DumbSnapshotter::default()),
707 TestLogger::stdout(),
708 );
709
710 assert!(result.is_err(), "Should return an error when no uploaders")
711 }
712
713 #[tokio::test]
714 async fn upload_immutable_archives_should_log_upload_errors() {
715 let (logger, log_inspector) = TestLogger::memory();
716 let mut uploader = MockImmutableFilesUploader::new();
717 uploader
718 .expect_batch_upload()
719 .return_once(|_, _| Err(anyhow!("Failure while uploading...")));
720
721 let builder = ImmutableArtifactBuilder::new(
722 get_builder_work_dir("upload_immutable_archives_should_log_upload_errors"),
723 vec![Arc::new(uploader)],
724 Arc::new(MockSnapshotter::new()),
725 logger,
726 )
727 .unwrap();
728
729 let _ = builder
730 .upload_immutable_archives(
731 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
732 CompressionAlgorithm::Gzip,
733 )
734 .await;
735
736 assert!(log_inspector.contains_log("Failure while uploading..."));
737 }
738
739 #[tokio::test]
740 async fn upload_immutable_archives_should_error_when_no_location_is_returned() {
741 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> =
742 vec![Arc::new(fake_uploader_returning_error())];
743
744 let builder = ImmutableArtifactBuilder::new(
745 get_builder_work_dir("upload_immutable_archives_should_error_when_no_location"),
746 uploaders,
747 Arc::new(MockSnapshotter::new()),
748 TestLogger::stdout(),
749 )
750 .unwrap();
751
752 let result = builder
753 .upload_immutable_archives(
754 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
755 CompressionAlgorithm::Gzip,
756 )
757 .await;
758
759 assert!(
760 result.is_err(),
761 "Should return an error when no location is returned"
762 );
763 }
764
765 #[tokio::test]
766 async fn upload_immutable_archives_should_return_location_even_with_uploaders_errors() {
767 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> = vec![
768 Arc::new(fake_uploader_returning_error()),
769 Arc::new(fake_uploader(
770 vec!["01.tar.gz", "02.tar.gz"],
771 "archive_2.tar.gz",
772 Some(CompressionAlgorithm::Gzip),
773 )),
774 Arc::new(fake_uploader_returning_error()),
775 ];
776
777 let builder = ImmutableArtifactBuilder::new(
778 get_builder_work_dir(
779 "upload_immutable_archives_should_return_location_even_with_uploaders_errors",
780 ),
781 uploaders,
782 Arc::new(MockSnapshotter::new()),
783 TestLogger::stdout(),
784 )
785 .unwrap();
786
787 let archive_paths = builder
788 .upload_immutable_archives(
789 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
790 CompressionAlgorithm::Gzip,
791 )
792 .await
793 .unwrap();
794
795 assert_equivalent!(
796 archive_paths,
797 vec![ImmutablesLocation::CloudStorage {
798 uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
799 compression_algorithm: Some(CompressionAlgorithm::Gzip),
800 }],
801 )
802 }
803
804 #[tokio::test]
805 async fn upload_immutable_archives_should_return_all_uploaders_returned_locations() {
806 let uploaders: Vec<Arc<dyn ImmutableFilesUploader>> = vec![
807 Arc::new(fake_uploader(
808 vec!["01.tar.gz", "02.tar.gz"],
809 "archive_1.tar.gz",
810 Some(CompressionAlgorithm::Gzip),
811 )),
812 Arc::new(fake_uploader(
813 vec!["01.tar.gz", "02.tar.gz"],
814 "archive_2.tar.gz",
815 Some(CompressionAlgorithm::Gzip),
816 )),
817 ];
818
819 let builder = ImmutableArtifactBuilder::new(
820 get_builder_work_dir(
821 "upload_immutable_archives_should_return_all_uploaders_returned_locations",
822 ),
823 uploaders,
824 Arc::new(MockSnapshotter::new()),
825 TestLogger::stdout(),
826 )
827 .unwrap();
828
829 let archive_paths = builder
830 .upload_immutable_archives(
831 &[PathBuf::from("01.tar.gz"), PathBuf::from("02.tar.gz")],
832 CompressionAlgorithm::Gzip,
833 )
834 .await
835 .unwrap();
836
837 assert_equivalent!(
838 archive_paths,
839 vec![
840 ImmutablesLocation::CloudStorage {
841 uri: MultiFilesUri::Template(TemplateUri("archive_1.tar.gz".to_string())),
842 compression_algorithm: Some(CompressionAlgorithm::Gzip),
843 },
844 ImmutablesLocation::CloudStorage {
845 uri: MultiFilesUri::Template(TemplateUri("archive_2.tar.gz".to_string())),
846 compression_algorithm: Some(CompressionAlgorithm::Gzip),
847 },
848 ],
849 )
850 }
851 }
852
853 mod batch_upload {
854 use mithril_common::test::TempDir;
855
856 use crate::file_uploaders::FileUploadRetryPolicy;
857 use crate::tools::url_sanitizer::SanitizedUrlWithTrailingSlash;
858
859 use super::*;
860
861 fn create_fake_archive(dir: &Path, name: &str) -> PathBuf {
862 let file_path = dir.join(name);
863 create_fake_file(
864 &file_path,
865 "I swear, this is an archive, not a temporary test file.",
866 );
867 file_path
868 }
869
870 #[tokio::test]
871 async fn extract_archive_name_to_deduce_template_location() {
872 let source_dir = TempDir::create(
873 "immutable",
874 "extract_archive_name_to_deduce_template_location_source",
875 );
876 let target_dir = TempDir::create(
877 "immutable",
878 "extract_archive_name_to_deduce_template_location_target",
879 );
880
881 let archive_1 = create_fake_archive(&source_dir, "00001.tar.gz");
882 let archive_2 = create_fake_archive(&source_dir, "00002.tar.gz");
883
884 let url_prefix =
885 SanitizedUrlWithTrailingSlash::parse("http://test.com:8080/base-root").unwrap();
886 let uploader = LocalUploader::new(
887 url_prefix,
888 &target_dir,
889 FileUploadRetryPolicy::never(),
890 TestLogger::stdout(),
891 );
892 let location = ImmutableFilesUploader::batch_upload(
893 &uploader,
894 &[archive_1.clone(), archive_2.clone()],
895 None,
896 )
897 .await
898 .expect("local upload should not fail");
899
900 assert!(target_dir.join(archive_1.file_name().unwrap()).exists());
901 assert!(target_dir.join(archive_2.file_name().unwrap()).exists());
902
903 let expected_location = ImmutablesLocation::CloudStorage {
904 uri: MultiFilesUri::Template(TemplateUri(
905 "http://test.com:8080/base-root/{immutable_file_number}.tar.gz".to_string(),
906 )),
907 compression_algorithm: None,
908 };
909 assert_eq!(expected_location, location);
910 }
911
912 #[tokio::test]
913 async fn returns_error_when_uploaded_filename_not_templatable_without_5_digits() {
914 let source_dir = TempDir::create(
915 "immutable",
916 "returns_error_when_uploaded_filename_not_templatable",
917 );
918 let target_dir = TempDir::create(
919 "immutable",
920 "returns_error_when_uploaded_filename_not_templatable",
921 );
922
923 let archive = create_fake_archive(&source_dir, "not-templatable.tar.gz");
924
925 let url_prefix =
926 SanitizedUrlWithTrailingSlash::parse("http://test.com:8080/base-root").unwrap();
927 let uploader = LocalUploader::new(
928 url_prefix,
929 &target_dir,
930 FileUploadRetryPolicy::never(),
931 TestLogger::stdout(),
932 );
933
934 ImmutableFilesUploader::batch_upload(&uploader, &[archive], None)
935 .await
936 .expect_err("Should return an error when not template found");
937 }
938 }
939
940 mod immutable_file_number_extractor {
941 use super::*;
942
943 #[test]
944 fn returns_none_when_not_templatable_without_5_digits() {
945 let template = immutable_file_number_extractor("not-templatable.tar.gz").unwrap();
946
947 assert!(template.is_none());
948 }
949
950 #[test]
951 fn returns_template() {
952 let template = immutable_file_number_extractor("http://whatever/00001.tar.gz").unwrap();
953
954 assert_eq!(
955 template,
956 Some("http://whatever/{immutable_file_number}.tar.gz".to_string())
957 );
958 }
959
960 #[test]
961 fn replaces_last_occurence_of_5_digits() {
962 let template =
963 immutable_file_number_extractor("http://00001/whatever/00001.tar.gz").unwrap();
964
965 assert_eq!(
966 template,
967 Some("http://00001/whatever/{immutable_file_number}.tar.gz".to_string())
968 );
969 }
970
971 #[test]
972 fn replaces_last_occurence_when_more_than_5_digits() {
973 let template =
974 immutable_file_number_extractor("http://whatever/123456789.tar.gz").unwrap();
975
976 assert_eq!(
977 template,
978 Some("http://whatever/1234{immutable_file_number}.tar.gz".to_string())
979 );
980 }
981 }
982
983 #[test]
984 fn compute_average_uncompressed_size() {
985 let total_size = 14;
986 let average_size_up_to_immutable_2 =
987 ImmutableArtifactBuilder::compute_average_uncompressed_size(total_size, 2);
988 assert_eq!(7, average_size_up_to_immutable_2);
989
990 let average_size_up_to_immutable_5 =
992 ImmutableArtifactBuilder::compute_average_uncompressed_size(total_size, 5);
993 assert_eq!(2, average_size_up_to_immutable_5);
994
995 let average_size_up_to_immutable_0 =
997 ImmutableArtifactBuilder::compute_average_uncompressed_size(total_size, 0);
998 assert_eq!(0, average_size_up_to_immutable_0);
999 }
1000}