mithril_cardano_node_internal_database/entities/
immutable_file.rs

1use digest::{Digest, Output};
2use std::{
3    cmp::Ordering,
4    fs::File,
5    io,
6    num::ParseIntError,
7    path::{Path, PathBuf},
8};
9use thiserror::Error;
10use walkdir::{DirEntry, WalkDir};
11
12use mithril_common::entities::{ImmutableFileName, ImmutableFileNumber};
13
14use crate::IMMUTABLE_DIR;
15use crate::entities::ImmutableFileListingError::{MissingImmutableFiles, MissingImmutableFolder};
16
17const IMMUTABLE_FILE_EXTENSIONS: [&str; 3] = ["chunk", "primary", "secondary"];
18
19fn is_immutable(entry: &walkdir::DirEntry) -> bool {
20    let is_file = entry.file_type().is_file();
21    let extension = entry.path().extension().map(|e| e.to_string_lossy());
22
23    is_file && extension.is_some_and(|e| IMMUTABLE_FILE_EXTENSIONS.contains(&e.as_ref()))
24}
25
26/// Walk the given path and return the first directory named "immutable" it finds
27fn find_immutables_dir(path_to_walk: &Path) -> Option<PathBuf> {
28    WalkDir::new(path_to_walk)
29        .into_iter()
30        .filter_entry(|e| e.file_type().is_dir())
31        .filter_map(|e| e.ok())
32        .find(|f| f.file_name() == IMMUTABLE_DIR)
33        .map(|e| e.into_path())
34}
35
36/// Walk the given immutable directory and return an iterator over its files (no subdirectories)
37fn walk_immutables_in_dir<P: AsRef<Path>>(immutable_dir: P) -> impl Iterator<Item = DirEntry> {
38    WalkDir::new(immutable_dir)
39        .min_depth(1)
40        .max_depth(1)
41        .into_iter()
42        .filter_entry(is_immutable)
43        .filter_map(|file| file.ok())
44}
45
46/// Represent an immutable file in a Cardano node database directory
47#[derive(Debug, PartialEq, Eq, Clone)]
48pub struct ImmutableFile {
49    /// The path to the immutable file
50    pub path: PathBuf,
51
52    /// The immutable file number
53    pub number: ImmutableFileNumber,
54
55    /// The filename
56    pub filename: ImmutableFileName,
57}
58
59/// [ImmutableFile::new] related errors.
60#[derive(Error, Debug)]
61pub enum ImmutableFileCreationError {
62    /// Raised when the immutable file stem extraction fails.
63    #[error("Couldn't extract the file stem for '{path:?}'")]
64    FileStemExtraction {
65        /// Path for which file stem extraction failed.
66        path: PathBuf,
67    },
68
69    /// Raised when the immutable file filename extraction fails.
70    #[error("Couldn't extract the filename as string for '{path:?}'")]
71    FileNameExtraction {
72        /// Path for which filename extraction failed.
73        path: PathBuf,
74    },
75
76    /// Raised when the immutable file number parsing, from the filename, fails.
77    #[error("Error while parsing immutable file number")]
78    FileNumberParsing(#[from] ParseIntError),
79}
80
81/// [ImmutableFile::list_completed_in_dir] related errors.
82#[derive(Error, Debug)]
83pub enum ImmutableFileListingError {
84    /// Raised when the metadata of a file could not be read.
85    #[error("metadata parsing failed")]
86    MetadataParsing(#[from] io::Error),
87
88    /// Raised when [ImmutableFile::new] fails.
89    #[error("immutable file creation error")]
90    ImmutableFileCreation(#[from] ImmutableFileCreationError),
91
92    /// Raised when the "immutable" folder could not be found in a file structure.
93    #[error("Couldn't find the 'immutable' folder in '{0:?}'")]
94    MissingImmutableFolder(PathBuf),
95
96    /// Raised when no immutable files could be found in the 'immutable' folder.
97    #[error("There are no immutable files in '{0:?}'")]
98    MissingImmutableFiles(PathBuf),
99}
100
101impl ImmutableFile {
102    /// ImmutableFile factory
103    pub fn new(path: PathBuf) -> Result<ImmutableFile, ImmutableFileCreationError> {
104        let filename = path
105            .file_name()
106            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?
107            .to_str()
108            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?
109            .to_string();
110
111        let filestem = path
112            .file_stem()
113            .ok_or(ImmutableFileCreationError::FileStemExtraction { path: path.clone() })?
114            .to_str()
115            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?;
116        let immutable_file_number = filestem.parse::<ImmutableFileNumber>()?;
117
118        Ok(Self {
119            path,
120            number: immutable_file_number,
121            filename,
122        })
123    }
124
125    /// Compute the hash of this immutable file.
126    pub fn compute_raw_hash<D>(&self) -> Result<Output<D>, io::Error>
127    where
128        D: Digest + io::Write,
129    {
130        let mut hasher = D::new();
131        let mut file = File::open(&self.path)?;
132        io::copy(&mut file, &mut hasher)?;
133        Ok(hasher.finalize())
134    }
135
136    /// List all [`ImmutableFile`] in a given directory.
137    pub fn list_all_in_dir(dir: &Path) -> Result<Vec<ImmutableFile>, ImmutableFileListingError> {
138        let immutable_dir = find_immutables_dir(dir).ok_or(
139            ImmutableFileListingError::MissingImmutableFolder(dir.to_path_buf()),
140        )?;
141        let mut files: Vec<ImmutableFile> = vec![];
142
143        for path in walk_immutables_in_dir(&immutable_dir) {
144            let immutable_file = ImmutableFile::new(path.into_path())?;
145            files.push(immutable_file);
146        }
147        files.sort();
148
149        Ok(files)
150    }
151
152    /// List all complete [`ImmutableFile`] in a given directory.
153    ///
154    /// Important Note: It will skip the last chunk / primary / secondary trio since they're not yet
155    /// complete.
156    pub fn list_completed_in_dir(
157        dir: &Path,
158    ) -> Result<Vec<ImmutableFile>, ImmutableFileListingError> {
159        let files = Self::list_all_in_dir(dir)?;
160
161        match files.last() {
162            // empty list
163            None => Ok(files),
164            // filter out the last immutable file(s)
165            Some(last_file) => {
166                let last_number = last_file.number;
167                Ok(files.into_iter().filter(|f| f.number < last_number).collect())
168            }
169        }
170    }
171
172    /// Check if at least one immutable file exists in the given directory
173    pub fn at_least_one_immutable_files_exist_in_dir(
174        dir: &Path,
175    ) -> Result<(), ImmutableFileListingError> {
176        let immutable_dir =
177            find_immutables_dir(dir).ok_or(MissingImmutableFolder(dir.to_path_buf()))?;
178        if walk_immutables_in_dir(immutable_dir).next().is_some() {
179            Ok(())
180        } else {
181            Err(MissingImmutableFiles(dir.to_path_buf().join(IMMUTABLE_DIR)))
182        }
183    }
184}
185
186impl PartialOrd for ImmutableFile {
187    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
188        Some(self.cmp(other))
189    }
190}
191
192impl Ord for ImmutableFile {
193    fn cmp(&self, other: &Self) -> Ordering {
194        self.number.cmp(&other.number).then(self.path.cmp(&other.path))
195    }
196}
197
198#[cfg(test)]
199mod tests {
200    use std::fs;
201    use std::io::prelude::*;
202
203    use mithril_common::temp_dir_create;
204    use mithril_common::test_utils::TempDir;
205
206    use super::*;
207
208    fn get_test_dir(subdir_name: &str) -> PathBuf {
209        TempDir::create("immutable_file", subdir_name)
210    }
211
212    fn create_fake_files(parent_dir: &Path, child_filenames: &[&str]) {
213        for filename in child_filenames {
214            let file = parent_dir.join(Path::new(filename));
215            let mut source_file = File::create(file).unwrap();
216            write!(source_file, "This is a test file named '{filename}'").unwrap();
217        }
218    }
219
220    fn extract_filenames(immutables: &[ImmutableFile]) -> Vec<String> {
221        immutables
222            .iter()
223            .map(|i| i.path.file_name().unwrap().to_str().unwrap().to_owned())
224            .collect()
225    }
226
227    #[test]
228    fn list_completed_immutable_file_fail_if_not_in_immutable_dir() {
229        let target_dir = get_test_dir("list_immutable_file_fail_if_not_in_immutable_dir/invalid");
230        let entries = vec![];
231        create_fake_files(&target_dir, &entries);
232
233        ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
234            .expect_err("ImmutableFile::list_in_dir should have Failed");
235    }
236
237    #[test]
238    fn list_all_immutable_file_should_not_skip_last_number() {
239        let target_dir =
240            get_test_dir("list_all_immutable_file_should_not_skip_last_number/immutable");
241        let entries = vec![
242            "123.chunk",
243            "123.primary",
244            "123.secondary",
245            "125.chunk",
246            "125.primary",
247            "125.secondary",
248            "0124.chunk",
249            "0124.primary",
250            "0124.secondary",
251            "223.chunk",
252            "223.primary",
253            "223.secondary",
254            "0423.chunk",
255            "0423.primary",
256            "0423.secondary",
257            "0424.chunk",
258            "0424.primary",
259            "0424.secondary",
260            "21.chunk",
261            "21.primary",
262            "21.secondary",
263        ];
264        create_fake_files(&target_dir, &entries);
265        let result = ImmutableFile::list_all_in_dir(target_dir.parent().unwrap())
266            .expect("ImmutableFile::list_in_dir Failed");
267
268        assert_eq!(result.last().unwrap().number, 424);
269        let expected_entries_length = 21;
270        assert_eq!(
271            expected_entries_length,
272            result.len(),
273            "Expected to find {} files but found {}",
274            entries.len(),
275            result.len(),
276        );
277    }
278
279    #[test]
280    fn list_completed_immutable_file_should_skip_last_number() {
281        let target_dir = get_test_dir("list_immutable_file_should_skip_last_number/immutable");
282        let entries = vec![
283            "123.chunk",
284            "123.primary",
285            "123.secondary",
286            "125.chunk",
287            "125.primary",
288            "125.secondary",
289            "0124.chunk",
290            "0124.primary",
291            "0124.secondary",
292            "223.chunk",
293            "223.primary",
294            "223.secondary",
295            "0423.chunk",
296            "0423.primary",
297            "0423.secondary",
298            "0424.chunk",
299            "0424.primary",
300            "0424.secondary",
301            "21.chunk",
302            "21.primary",
303            "21.secondary",
304        ];
305        create_fake_files(&target_dir, &entries);
306        let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
307            .expect("ImmutableFile::list_in_dir Failed");
308
309        assert_eq!(result.last().unwrap().number, 423);
310        assert_eq!(
311            result.len(),
312            entries.len() - 3,
313            "Expected to find {} files since the last (chunk, primary, secondary) trio is skipped, but found {}",
314            entries.len() - 3,
315            result.len(),
316        );
317    }
318
319    #[test]
320    fn list_completed_immutable_file_should_works_in_a_empty_folder() {
321        let target_dir =
322            get_test_dir("list_immutable_file_should_works_even_in_a_empty_folder/immutable");
323        let entries = vec![];
324        create_fake_files(&target_dir, &entries);
325        let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
326            .expect("ImmutableFile::list_in_dir Failed");
327
328        assert!(result.is_empty());
329    }
330
331    #[test]
332    fn list_completed_immutable_file_order_should_be_deterministic() {
333        let target_dir =
334            get_test_dir("list_completed_immutable_file_order_should_be_deterministic/immutable");
335        let entries = vec![
336            "21.chunk",
337            "21.primary",
338            "21.secondary",
339            "123.chunk",
340            "123.primary",
341            "123.secondary",
342            "124.chunk",
343            "124.primary",
344            "124.secondary",
345            "125.chunk",
346            "125.primary",
347            "125.secondary",
348            "223.chunk",
349            "223.primary",
350            "223.secondary",
351            "423.chunk",
352            "423.primary",
353            "423.secondary",
354            "424.chunk",
355            "424.primary",
356            "424.secondary",
357        ];
358        create_fake_files(&target_dir, &entries);
359        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
360            .expect("ImmutableFile::list_in_dir Failed");
361        let immutables_names: Vec<String> = extract_filenames(&immutables);
362
363        let expected: Vec<&str> = entries.into_iter().rev().skip(3).rev().collect();
364        assert_eq!(expected, immutables_names);
365    }
366
367    #[test]
368    fn list_completed_immutable_file_should_work_with_non_immutable_files() {
369        let target_dir =
370            get_test_dir("list_immutable_file_should_work_with_non_immutable_files/immutable");
371        let entries = vec![
372            "123.chunk",
373            "123.primary",
374            "123.secondary",
375            "124.chunk",
376            "124.primary",
377            "124.secondary",
378            "README.md",
379            "124.secondary.back",
380        ];
381        create_fake_files(&target_dir, &entries);
382        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
383            .expect("ImmutableFile::list_in_dir Failed");
384        let immutables_names: Vec<String> = extract_filenames(&immutables);
385
386        let expected: Vec<&str> = entries.into_iter().rev().skip(5).rev().collect();
387        assert_eq!(expected, immutables_names);
388    }
389
390    #[test]
391    fn list_completed_immutable_file_can_list_incomplete_trio() {
392        let target_dir = get_test_dir("list_immutable_file_can_list_incomplete_trio/immutable");
393        let entries = vec![
394            "21.chunk",
395            "21.primary",
396            "21.secondary",
397            "123.chunk",
398            "123.secondary",
399            "124.chunk",
400            "124.primary",
401            "125.primary",
402            "125.secondary",
403            "223.chunk",
404            "224.primary",
405            "225.secondary",
406            "226.chunk",
407        ];
408        create_fake_files(&target_dir, &entries);
409        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
410            .expect("ImmutableFile::list_in_dir Failed");
411        let immutables_names: Vec<String> = extract_filenames(&immutables);
412
413        let expected: Vec<&str> = entries.into_iter().rev().skip(1).rev().collect();
414        assert_eq!(expected, immutables_names);
415    }
416
417    #[test]
418    fn at_least_one_immutable_files_exist_in_dir_throw_error_if_immutable_dir_does_not_exist() {
419        let database_path = temp_dir_create!();
420
421        let error = ImmutableFile::at_least_one_immutable_files_exist_in_dir(&database_path)
422            .expect_err("check_presence_of_immutables should fail");
423        assert_eq!(
424            error.to_string(),
425            format!("Couldn't find the 'immutable' folder in '{database_path:?}'")
426        );
427    }
428
429    #[test]
430    fn at_least_one_immutable_files_exist_in_dir_throw_error_if_immutable_dir_is_empty() {
431        let database_path = temp_dir_create!();
432        fs::create_dir(database_path.join(IMMUTABLE_DIR)).unwrap();
433
434        let error = ImmutableFile::at_least_one_immutable_files_exist_in_dir(&database_path)
435            .expect_err("check_presence_of_immutables should fail");
436        assert_eq!(
437            error.to_string(),
438            format!(
439                "There are no immutable files in '{:?}'",
440                database_path.join(IMMUTABLE_DIR)
441            )
442        );
443    }
444
445    #[test]
446    fn at_least_one_immutable_files_exist_in_dir_is_ok_if_immutable_dir_contains_at_least_one_file()
447    {
448        let database_dir = temp_dir_create!();
449        let database_path = database_dir.as_path();
450        let immutable_file_path = database_dir.join(IMMUTABLE_DIR).join("00001.chunk");
451        fs::create_dir(database_dir.join(IMMUTABLE_DIR)).unwrap();
452        File::create(immutable_file_path).unwrap();
453
454        ImmutableFile::at_least_one_immutable_files_exist_in_dir(database_path)
455            .expect("check_presence_of_immutables should succeed");
456    }
457}
mithril_cardano_node_internal_database/entities/immutable_file.rs

mithril_cardano_node_internal_database/entities/
immutable_file.rs
