1use anyhow::Context;
105#[cfg(feature = "fs")]
106use slog::Logger;
107#[cfg(feature = "fs")]
108use std::path::{Path, PathBuf};
109use std::sync::Arc;
110use thiserror::Error;
111
112#[cfg(feature = "fs")]
113use mithril_common::entities::CompressionAlgorithm;
114
115use crate::aggregator_client::{AggregatorClient, AggregatorClientError, AggregatorRequest};
116#[cfg(feature = "fs")]
117use crate::feedback::FeedbackSender;
118#[cfg(feature = "fs")]
119use crate::file_downloader::{DownloadEvent, FileDownloader};
120#[cfg(feature = "fs")]
121use crate::utils::create_bootstrap_node_files;
122#[cfg(feature = "fs")]
123use crate::utils::AncillaryVerifier;
124use crate::{MithrilResult, Snapshot, SnapshotListItem};
125
126#[derive(Error, Debug)]
128pub enum SnapshotClientError {
129 #[error("Could not find a working download location for the snapshot digest '{digest}', tried location: {{'{locations}'}}.")]
131 NoWorkingLocation {
132 digest: String,
134
135 locations: String,
137 },
138 #[error("Ancillary verifier is not set, please use `set_ancillary_verification_key` when creating the client")]
140 MissingAncillaryVerifier,
141}
142
143pub struct SnapshotClient {
145 aggregator_client: Arc<dyn AggregatorClient>,
146 #[cfg(feature = "fs")]
147 http_file_downloader: Arc<dyn FileDownloader>,
148 #[cfg(feature = "fs")]
149 ancillary_verifier: Option<Arc<AncillaryVerifier>>,
150 #[cfg(feature = "fs")]
151 _feedback_sender: FeedbackSender,
152 #[cfg(feature = "fs")]
153 logger: Logger,
154}
155
156impl SnapshotClient {
157 pub fn new(
159 aggregator_client: Arc<dyn AggregatorClient>,
160 #[cfg(feature = "fs")] http_file_downloader: Arc<dyn FileDownloader>,
161 #[cfg(feature = "fs")] ancillary_verifier: Option<Arc<AncillaryVerifier>>,
162 #[cfg(feature = "fs")] feedback_sender: FeedbackSender,
163 #[cfg(feature = "fs")] logger: Logger,
164 ) -> Self {
165 Self {
166 aggregator_client,
167 #[cfg(feature = "fs")]
168 http_file_downloader,
169 #[cfg(feature = "fs")]
170 ancillary_verifier,
171 #[cfg(feature = "fs")]
173 _feedback_sender: feedback_sender,
174 #[cfg(feature = "fs")]
175 logger: mithril_common::logging::LoggerExtensions::new_with_component_name::<Self>(
176 &logger,
177 ),
178 }
179 }
180
181 pub async fn list(&self) -> MithrilResult<Vec<SnapshotListItem>> {
183 let response = self
184 .aggregator_client
185 .get_content(AggregatorRequest::ListSnapshots)
186 .await
187 .with_context(|| "Snapshot Client can not get the artifact list")?;
188 let items = serde_json::from_str::<Vec<SnapshotListItem>>(&response)
189 .with_context(|| "Snapshot Client can not deserialize artifact list")?;
190
191 Ok(items)
192 }
193
194 pub async fn get(&self, digest: &str) -> MithrilResult<Option<Snapshot>> {
196 match self
197 .aggregator_client
198 .get_content(AggregatorRequest::GetSnapshot {
199 digest: digest.to_string(),
200 })
201 .await
202 {
203 Ok(content) => {
204 let snapshot: Snapshot = serde_json::from_str(&content)
205 .with_context(|| "Snapshot Client can not deserialize artifact")?;
206
207 Ok(Some(snapshot))
208 }
209 Err(AggregatorClientError::RemoteServerLogical(_)) => Ok(None),
210 Err(e) => Err(e.into()),
211 }
212 }
213
214 cfg_fs! {
215 pub async fn download_unpack_full(
223 &self,
224 snapshot: &Snapshot,
225 target_dir: &Path,
226 ) -> MithrilResult<()> {
227 use crate::feedback::MithrilEvent;
228 if self.ancillary_verifier.is_none() {
229 return Err(SnapshotClientError::MissingAncillaryVerifier.into());
230 }
231
232 let download_id = MithrilEvent::new_snapshot_download_id();
233 self.download_unpack_immutables_files(snapshot, target_dir, &download_id)
234 .await?;
235 self.download_unpack_ancillary(snapshot, target_dir, &download_id)
236 .await?;
237 create_bootstrap_node_files(
238 &self.logger,
239 target_dir,
240 &snapshot.network,
241 )?;
242
243 Ok(())
244 }
245
246 pub async fn download_unpack(
254 &self,
255 snapshot: &Snapshot,
256 target_dir: &Path,
257 ) -> MithrilResult<()> {
258 use crate::feedback::MithrilEvent;
259 let download_id = MithrilEvent::new_snapshot_download_id();
260 self.download_unpack_immutables_files(snapshot, target_dir, &download_id)
261 .await?;
262 create_bootstrap_node_files(
263 &self.logger,
264 target_dir,
265 &snapshot.network,
266 )?;
267
268 Ok(())
269 }
270
271 async fn download_unpack_immutables_files(
272 &self,
273 snapshot: &Snapshot,
274 target_dir: &Path,
275 download_id: &str,
276 ) -> MithrilResult<()> {
277 self.download_unpack_file(
278 &snapshot.digest,
279 &snapshot.locations,
280 snapshot.size,
281 target_dir,
282 snapshot.compression_algorithm,
283 DownloadEvent::Full {
284 download_id: download_id.to_string(),
285 digest: snapshot.digest.clone(),
286 },
287 )
288 .await?;
289
290 Ok(())
291 }
292
293 async fn download_unpack_ancillary(
294 &self,
295 snapshot: &Snapshot,
296 target_dir: &Path,
297 download_id: &str,
298 ) -> MithrilResult<()> {
299 slog::info!(
300 self.logger,
301 "Ancillary verification doesn't use the Mithril certification: it is done with a signature made with an IOG owned key."
302 );
303
304 match &snapshot.ancillary_locations {
305 None => Ok(()),
306 Some(ancillary_locations) => {
307 let temp_ancillary_unpack_dir = Self::ancillary_subdir(target_dir, download_id);
308 tokio::fs::create_dir(&temp_ancillary_unpack_dir)
309 .await
310 .with_context(|| {
311 format!(
312 "Snapshot Client can not create ancillary unpack directory '{}'",
313 temp_ancillary_unpack_dir.display()
314 )
315 })?;
316
317 let result = self
318 .download_unpack_verify_ancillary(
319 snapshot,
320 ancillary_locations,
321 snapshot.ancillary_size.unwrap_or(0),
322 target_dir,
323 &temp_ancillary_unpack_dir,
324 download_id,
325 )
326 .await;
327
328 if let Err(e) = std::fs::remove_dir_all(&temp_ancillary_unpack_dir) {
329 slog::warn!(
330 self.logger, "Failed to remove ancillary unpack directory '{}'", temp_ancillary_unpack_dir.display();
331 "error" => ?e
332 );
333 }
334
335 result
336 }
337 }
338 }
339
340 async fn download_unpack_verify_ancillary(
341 &self,
342 snapshot: &Snapshot,
343 ancillary_locations: &[String],
344 ancillary_size: u64,
345 target_dir: &Path,
346 temp_ancillary_unpack_dir: &Path,
347 download_id: &str,
348 ) -> MithrilResult<()> {
349 self.download_unpack_file(
350 &snapshot.digest,
351 ancillary_locations,
352 ancillary_size,
353 temp_ancillary_unpack_dir,
354 snapshot.compression_algorithm,
355 DownloadEvent::FullAncillary {
356 download_id: download_id.to_string(),
357 },
358 )
359 .await?;
360
361 let ancillary_verifier = self
362 .ancillary_verifier
363 .as_ref()
364 .ok_or(SnapshotClientError::MissingAncillaryVerifier)?;
365
366 let validated_manifest = ancillary_verifier.verify(temp_ancillary_unpack_dir).await?;
367 validated_manifest
368 .move_to_final_location(target_dir)
369 .await?;
370
371 Ok(())
372 }
373
374 async fn download_unpack_file(
375 &self,
376 digest: &str,
377 locations: &[String],
378 size: u64,
379 target_dir: &Path,
380 compression_algorithm: CompressionAlgorithm,
381 download_event: DownloadEvent,
382 ) -> MithrilResult<()> {
383 for location in locations {
384 let file_downloader_uri = location.to_owned().into();
385
386 if let Err(error) = self
387 .http_file_downloader
388 .download_unpack(
389 &file_downloader_uri,
390 size,
391 target_dir,
392 Some(compression_algorithm),
393 download_event.clone(),
394 )
395 .await
396 {
397 slog::warn!(self.logger, "Failed downloading snapshot from '{location}'"; "error" => ?error);
398 } else {
399 return Ok(());
400 }
401 }
402
403 let locations = locations.join(", ");
404
405 Err(SnapshotClientError::NoWorkingLocation {
406 digest: digest.to_string(),
407 locations,
408 }
409 .into())
410 }
411
412 fn ancillary_subdir(target_dir: &Path, download_id: &str) -> PathBuf {
413 target_dir.join(format!("ancillary-{download_id}"))
414 }
415 }
416
417 pub async fn add_statistics(&self, snapshot: &Snapshot) -> MithrilResult<()> {
419 let _response = self
420 .aggregator_client
421 .post_content(AggregatorRequest::IncrementSnapshotStatistic {
422 snapshot: serde_json::to_string(snapshot)?,
423 })
424 .await?;
425
426 Ok(())
427 }
428}
429
430#[cfg(all(test, feature = "fs"))]
431mod tests {
432 use crate::{
433 aggregator_client::MockAggregatorClient,
434 common::CompressionAlgorithm,
435 feedback::MithrilEvent,
436 file_downloader::{MockFileDownloader, MockFileDownloaderBuilder},
437 test_utils::TestLogger,
438 };
439
440 use super::*;
441
442 use mithril_common::{assert_dir_eq, temp_dir_create};
443
444 fn dummy_download_event() -> DownloadEvent {
445 DownloadEvent::Full {
446 download_id: MithrilEvent::new_snapshot_download_id(),
447 digest: "test-digest".to_string(),
448 }
449 }
450
451 fn setup_snapshot_client(
452 file_downloader: Arc<dyn FileDownloader>,
453 ancillary_verifier: Option<Arc<AncillaryVerifier>>,
454 ) -> SnapshotClient {
455 let aggregator_client = Arc::new(MockAggregatorClient::new());
456 let logger = TestLogger::stdout();
457
458 SnapshotClient::new(
459 aggregator_client,
460 file_downloader,
461 ancillary_verifier,
462 FeedbackSender::new(&[]),
463 logger.clone(),
464 )
465 }
466
467 mod download_unpack_file {
468 use super::*;
469
470 fn setup_snapshot_client(file_downloader: Arc<dyn FileDownloader>) -> SnapshotClient {
471 super::setup_snapshot_client(file_downloader, None)
472 }
473
474 #[tokio::test]
475 async fn log_warning_if_location_fails() {
476 let (logger, log_inspector) = TestLogger::memory();
477 let mock_downloader = MockFileDownloaderBuilder::default()
478 .with_file_uri("http://whatever.co/snapshot")
479 .with_failure()
480 .build();
481
482 let client = SnapshotClient {
483 logger,
484 ..setup_snapshot_client(Arc::new(mock_downloader))
485 };
486
487 let _result = client
488 .download_unpack_file(
489 "test-digest",
490 &["http://whatever.co/snapshot".to_string()],
491 19,
492 &PathBuf::from("/whatever"),
493 CompressionAlgorithm::Gzip,
494 dummy_download_event(),
495 )
496 .await;
497
498 assert!(
499 log_inspector.contains_log("Failed downloading snapshot"),
500 "Expected log message not found, logs: {log_inspector}"
501 );
502 }
503
504 #[tokio::test]
505 async fn error_contains_list_of_all_tried_locations_if_all_attempts_fails() {
506 let test_locations = vec![
507 "http://example.com/snapshot1".to_string(),
508 "http://example.com/snapshot2".to_string(),
509 ];
510 let mock_downloader = MockFileDownloaderBuilder::default()
511 .with_file_uri("http://example.com/snapshot1")
512 .with_failure()
513 .next_call()
514 .with_file_uri("http://example.com/snapshot2")
515 .with_failure()
516 .build();
517 let client = setup_snapshot_client(Arc::new(mock_downloader));
518
519 let error = client
520 .download_unpack_file(
521 "test-digest",
522 &test_locations,
523 19,
524 &PathBuf::from("/whatever"),
525 CompressionAlgorithm::Gzip,
526 dummy_download_event(),
527 )
528 .await
529 .expect_err("Should fail when all locations fail");
530
531 if let Some(SnapshotClientError::NoWorkingLocation { digest, locations }) =
532 error.downcast_ref::<SnapshotClientError>()
533 {
534 assert_eq!(digest, "test-digest");
535 assert_eq!(
536 locations,
537 "http://example.com/snapshot1, http://example.com/snapshot2"
538 );
539 } else {
540 panic!("Expected SnapshotClientError::NoWorkingLocation, but got: {error:?}");
541 }
542 }
543
544 #[tokio::test]
545 async fn fallback_to_another_location() {
546 let mock_downloader = MockFileDownloaderBuilder::default()
547 .with_file_uri("http://example.com/snapshot1")
548 .with_failure()
549 .next_call()
550 .with_file_uri("http://example.com/snapshot2")
551 .with_success()
552 .build();
553 let client = setup_snapshot_client(Arc::new(mock_downloader));
554
555 client
556 .download_unpack_file(
557 "test-digest",
558 &[
559 "http://example.com/snapshot1".to_string(),
560 "http://example.com/snapshot2".to_string(),
561 ],
562 19,
563 &PathBuf::from("/whatever"),
564 CompressionAlgorithm::Gzip,
565 dummy_download_event(),
566 )
567 .await
568 .expect("Should succeed when fallbacking to another location");
569 }
570
571 #[tokio::test]
572 async fn fail_if_location_list_is_empty() {
573 let client = setup_snapshot_client(Arc::new(MockFileDownloader::new()));
574
575 let error = client
576 .download_unpack_file(
577 "test-digest",
578 &Vec::new(),
579 19,
580 &PathBuf::from("/whatever"),
581 CompressionAlgorithm::Gzip,
582 dummy_download_event(),
583 )
584 .await
585 .expect_err("Should fail with empty location list");
586
587 if let Some(SnapshotClientError::NoWorkingLocation { locations, .. }) =
588 error.downcast_ref::<SnapshotClientError>()
589 {
590 assert_eq!(locations, "");
591 } else {
592 panic!("Expected SnapshotClientError::NoWorkingLocation, but got: {error:?}");
593 }
594 }
595 }
596
597 mod download_unpack_full {
598 use super::*;
599
600 #[tokio::test]
601 async fn fail_if_ancillary_verifier_is_not_set() {
602 let snapshot = Snapshot {
603 ancillary_locations: Some(vec!["http://example.com/ancillary".to_string()]),
604 ancillary_size: Some(123),
605 ..Snapshot::dummy()
606 };
607
608 let client = setup_snapshot_client(Arc::new(MockFileDownloader::new()), None);
609
610 let error = client
611 .download_unpack_full(&snapshot, &PathBuf::from("/whatever"))
612 .await
613 .expect_err("Should fail when ancillary verifier is not set");
614
615 assert!(
616 matches!(
617 error.downcast_ref::<SnapshotClientError>(),
618 Some(SnapshotClientError::MissingAncillaryVerifier)
619 ),
620 "Expected SnapshotClientError::MissingAncillaryVerifier, but got: {error:#?}"
621 );
622 }
623 }
624
625 mod download_unpack_ancillary {
626 use mithril_common::crypto_helper::ManifestSigner;
627 use mithril_common::test_utils::fake_keys;
628
629 use crate::file_downloader::FakeAncillaryFileBuilder;
630
631 use super::*;
632
633 #[tokio::test]
634 async fn log_a_info_message_telling_that_the_feature_does_not_use_mithril_certification() {
635 let (logger, log_inspector) = TestLogger::memory();
636 let verification_key = fake_keys::manifest_verification_key()[0]
637 .try_into()
638 .unwrap();
639 let snapshot = Snapshot {
640 ancillary_locations: None,
641 ancillary_size: None,
642 ..Snapshot::dummy()
643 };
644
645 let client = SnapshotClient {
646 logger,
647 ..setup_snapshot_client(
648 Arc::new(MockFileDownloader::new()),
649 Some(Arc::new(AncillaryVerifier::new(verification_key))),
650 )
651 };
652
653 client
654 .download_unpack_ancillary(
655 &snapshot,
656 &PathBuf::from("/whatever"),
657 "test-download-id",
658 )
659 .await
660 .unwrap();
661
662 assert!(
663 log_inspector.contains_log(
664 "Ancillary verification doesn't use the Mithril certification: it is \
665 done with a signature made with an IOG owned key."
666 ),
667 "Expected log message not found, logs: {log_inspector}"
668 );
669 }
670
671 #[tokio::test]
672 async fn do_nothing_if_no_ancillary_locations_available_in_snapshot() {
673 let verification_key = fake_keys::manifest_verification_key()[0]
674 .try_into()
675 .unwrap();
676 let snapshot = Snapshot {
677 ancillary_locations: None,
678 ancillary_size: None,
679 ..Snapshot::dummy()
680 };
681
682 let client = setup_snapshot_client(
683 Arc::new(MockFileDownloader::new()),
684 Some(Arc::new(AncillaryVerifier::new(verification_key))),
685 );
686
687 client
688 .download_unpack_ancillary(
689 &snapshot,
690 &PathBuf::from("/whatever"),
691 "test-download-id",
692 )
693 .await
694 .expect("Should succeed when no ancillary locations are available");
695 }
696
697 #[tokio::test]
698 async fn delete_temporary_unpack_subfolder_if_download_fail() {
699 let test_dir = temp_dir_create!();
700 let mock_downloader = MockFileDownloaderBuilder::default()
701 .with_file_uri("http://example.com/ancillary")
702 .with_failure()
703 .build();
704 let verification_key = fake_keys::manifest_verification_key()[0]
705 .try_into()
706 .unwrap();
707
708 let client = setup_snapshot_client(
709 Arc::new(mock_downloader),
710 Some(Arc::new(AncillaryVerifier::new(verification_key))),
711 );
712 let snapshot = Snapshot {
713 ancillary_locations: Some(vec!["http://example.com/ancillary".to_string()]),
714 ancillary_size: Some(123),
715 ..Snapshot::dummy()
716 };
717
718 client
719 .download_unpack_ancillary(&snapshot, &test_dir, "test-download-id")
720 .await
721 .unwrap_err();
722
723 assert!(!SnapshotClient::ancillary_subdir(&test_dir, "test-download-id").exists());
724 }
725
726 #[tokio::test]
727 async fn delete_temporary_unpack_subfolder_if_verify_fail() {
728 let test_dir = temp_dir_create!();
729 let mock_downloader = MockFileDownloaderBuilder::default()
730 .with_file_uri("http://example.com/ancillary")
731 .with_success()
732 .build();
733 let verification_key = fake_keys::manifest_verification_key()[0]
734 .try_into()
735 .unwrap();
736
737 let client = setup_snapshot_client(
738 Arc::new(mock_downloader),
739 Some(Arc::new(AncillaryVerifier::new(verification_key))),
740 );
741 let snapshot = Snapshot {
742 ancillary_locations: Some(vec!["http://example.com/ancillary".to_string()]),
743 ancillary_size: Some(123),
744 ..Snapshot::dummy()
745 };
746
747 client
748 .download_unpack_ancillary(&snapshot, &test_dir, "test-download-id")
749 .await
750 .unwrap_err();
751
752 assert!(!SnapshotClient::ancillary_subdir(&test_dir, "test-download-id").exists());
753 }
754
755 #[tokio::test]
756 async fn move_file_in_manifest_then_delete_temporary_unpack_subfolder_if_verify_succeed() {
757 let test_dir = temp_dir_create!();
758 let ancillary_signer = ManifestSigner::create_deterministic_signer();
759 let verification_key = ancillary_signer.verification_key();
760 let mock_downloader = MockFileDownloaderBuilder::default()
761 .with_file_uri("http://example.com/ancillary")
762 .with_success_and_create_fake_ancillary_files(
763 FakeAncillaryFileBuilder::builder()
764 .files_in_manifest_to_create(vec!["dummy_ledger".to_string()])
765 .files_not_in_manifest_to_create(vec!["not_in_ancillary".to_string()])
766 .sign_manifest(ancillary_signer)
767 .build(),
768 )
769 .build();
770
771 let client = setup_snapshot_client(
772 Arc::new(mock_downloader),
773 Some(Arc::new(AncillaryVerifier::new(verification_key))),
774 );
775
776 let snapshot = Snapshot {
777 ancillary_locations: Some(vec!["http://example.com/ancillary".to_string()]),
778 ancillary_size: Some(123),
779 ..Snapshot::dummy()
780 };
781 client
782 .download_unpack_ancillary(&snapshot, &test_dir, "test-download-id")
783 .await
784 .expect("Should succeed when ancillary verification is successful");
785
786 assert_dir_eq!(&test_dir, "* dummy_ledger");
787 }
788 }
789}