mithril_common/digesters/
immutable_file.rs

Help
1use crate::entities::{ImmutableFileName, ImmutableFileNumber};
2
3use crate::digesters::ImmutableFileListingError::MissingImmutableFolder;
4use digest::{Digest, Output};
5use std::{
6    cmp::Ordering,
7    fs::File,
8    io,
9    num::ParseIntError,
10    path::{Path, PathBuf},
11};
12use thiserror::Error;
13use walkdir::WalkDir;
14
15const IMMUTABLE_FILE_EXTENSIONS: [&str; 3] = ["chunk", "primary", "secondary"];
16
17fn is_immutable(entry: &walkdir::DirEntry) -> bool {
18    let is_file = entry.file_type().is_file();
19    let extension = entry.path().extension().map(|e| e.to_string_lossy());
20
21    is_file && extension.is_some_and(|e| IMMUTABLE_FILE_EXTENSIONS.contains(&e.as_ref()))
22}
23
24/// Walk the given path and return the first directory named "immutable" it finds
25fn find_immutables_dir(path_to_walk: &Path) -> Option<PathBuf> {
26    WalkDir::new(path_to_walk)
27        .into_iter()
28        .filter_entry(|e| e.file_type().is_dir())
29        .filter_map(|e| e.ok())
30        .find(|f| f.file_name() == "immutable")
31        .map(|e| e.into_path())
32}
33
34/// Represent an immutable file in a Cardano node database directory
35#[derive(Debug, PartialEq, Eq, Clone)]
36pub struct ImmutableFile {
37    /// The path to the immutable file
38    pub path: PathBuf,
39
40    /// The immutable file number
41    pub number: ImmutableFileNumber,
42
43    /// The filename
44    pub filename: ImmutableFileName,
45}
46
47/// [ImmutableFile::new] related errors.
48#[derive(Error, Debug)]
49pub enum ImmutableFileCreationError {
50    /// Raised when the immutable file stem extraction fails.
51    #[error("Couldn't extract the file stem for '{path:?}'")]
52    FileStemExtraction {
53        /// Path for which file stem extraction failed.
54        path: PathBuf,
55    },
56
57    /// Raised when the immutable file filename extraction fails.
58    #[error("Couldn't extract the filename as string for '{path:?}'")]
59    FileNameExtraction {
60        /// Path for which filename extraction failed.
61        path: PathBuf,
62    },
63
64    /// Raised when the immutable file number parsing, from the filename, fails.
65    #[error("Error while parsing immutable file number")]
66    FileNumberParsing(#[from] ParseIntError),
67}
68
69/// [ImmutableFile::list_completed_in_dir] related errors.
70#[derive(Error, Debug)]
71pub enum ImmutableFileListingError {
72    /// Raised when the metadata of a file could not be read.
73    #[error("metadata parsing failed")]
74    MetadataParsing(#[from] io::Error),
75
76    /// Raised when [ImmutableFile::new] fails.
77    #[error("immutable file creation error")]
78    ImmutableFileCreation(#[from] ImmutableFileCreationError),
79
80    /// Raised when the "immutable" folder could not be found in a file structure.
81    #[error("Couldn't find the 'immutable' folder in '{0:?}'")]
82    MissingImmutableFolder(PathBuf),
83}
84
85impl ImmutableFile {
86    /// ImmutableFile factory
87    pub fn new(path: PathBuf) -> Result<ImmutableFile, ImmutableFileCreationError> {
88        let filename = path
89            .file_name()
90            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?
91            .to_str()
92            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?
93            .to_string();
94
95        let filestem = path
96            .file_stem()
97            .ok_or(ImmutableFileCreationError::FileStemExtraction { path: path.clone() })?
98            .to_str()
99            .ok_or(ImmutableFileCreationError::FileNameExtraction { path: path.clone() })?;
100        let immutable_file_number = filestem.parse::<ImmutableFileNumber>()?;
101
102        Ok(Self {
103            path,
104            number: immutable_file_number,
105            filename,
106        })
107    }
108
109    cfg_test_tools! {
110        /// ImmutableFile factory, TEST ONLY as it bypass the checks done by [ImmutableFile::new].
111        pub fn dummy<T: Into<String>>(path: PathBuf, number: ImmutableFileNumber, filename: T) -> Self {
112            Self {
113                path,
114                number,
115                filename: filename.into(),
116            }
117        }
118    }
119
120    /// Compute the hash of this immutable file.
121    pub fn compute_raw_hash<D>(&self) -> Result<Output<D>, io::Error>
122    where
123        D: Digest + io::Write,
124    {
125        let mut hasher = D::new();
126        let mut file = File::open(&self.path)?;
127        io::copy(&mut file, &mut hasher)?;
128        Ok(hasher.finalize())
129    }
130
131    /// List all [`ImmutableFile`] in a given directory.
132    pub fn list_all_in_dir(dir: &Path) -> Result<Vec<ImmutableFile>, ImmutableFileListingError> {
133        let immutable_dir =
134            find_immutables_dir(dir).ok_or(MissingImmutableFolder(dir.to_path_buf()))?;
135        let mut files: Vec<ImmutableFile> = vec![];
136
137        for path in WalkDir::new(immutable_dir)
138            .min_depth(1)
139            .max_depth(1)
140            .into_iter()
141            .filter_entry(is_immutable)
142            .filter_map(|file| file.ok())
143        {
144            let immutable_file = ImmutableFile::new(path.into_path())?;
145            files.push(immutable_file);
146        }
147        files.sort();
148
149        Ok(files)
150    }
151
152    /// List all complete [`ImmutableFile`] in a given directory.
153    ///
154    /// Important Note: It will skip the last chunk / primary / secondary trio since they're not yet
155    /// complete.
156    pub fn list_completed_in_dir(
157        dir: &Path,
158    ) -> Result<Vec<ImmutableFile>, ImmutableFileListingError> {
159        let files = Self::list_all_in_dir(dir)?;
160
161        match files.last() {
162            // empty list
163            None => Ok(files),
164            // filter out the last immutable file(s)
165            Some(last_file) => {
166                let last_number = last_file.number;
167                Ok(files
168                    .into_iter()
169                    .filter(|f| f.number < last_number)
170                    .collect())
171            }
172        }
173    }
174}
175
176impl PartialOrd for ImmutableFile {
177    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
178        Some(self.cmp(other))
179    }
180}
181
182impl Ord for ImmutableFile {
183    fn cmp(&self, other: &Self) -> Ordering {
184        self.number
185            .cmp(&other.number)
186            .then(self.path.cmp(&other.path))
187    }
188}
189
190#[cfg(test)]
191mod tests {
192    use super::ImmutableFile;
193    use crate::test_utils::TempDir;
194    use std::fs::File;
195    use std::io::prelude::*;
196    use std::path::{Path, PathBuf};
197
198    fn get_test_dir(subdir_name: &str) -> PathBuf {
199        TempDir::create("immutable_file", subdir_name)
200    }
201
202    fn create_fake_files(parent_dir: &Path, child_filenames: &[&str]) {
203        for filename in child_filenames {
204            let file = parent_dir.join(Path::new(filename));
205            let mut source_file = File::create(file).unwrap();
206            write!(source_file, "This is a test file named '{filename}'").unwrap();
207        }
208    }
209
210    fn extract_filenames(immutables: &[ImmutableFile]) -> Vec<String> {
211        immutables
212            .iter()
213            .map(|i| i.path.file_name().unwrap().to_str().unwrap().to_owned())
214            .collect()
215    }
216
217    #[test]
218    fn list_completed_immutable_file_fail_if_not_in_immutable_dir() {
219        let target_dir = get_test_dir("list_immutable_file_fail_if_not_in_immutable_dir/invalid");
220        let entries = vec![];
221        create_fake_files(&target_dir, &entries);
222
223        ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
224            .expect_err("ImmutableFile::list_in_dir should have Failed");
225    }
226
227    #[test]
228    fn list_all_immutable_file_should_not_skip_last_number() {
229        let target_dir =
230            get_test_dir("list_all_immutable_file_should_not_skip_last_number/immutable");
231        let entries = vec![
232            "123.chunk",
233            "123.primary",
234            "123.secondary",
235            "125.chunk",
236            "125.primary",
237            "125.secondary",
238            "0124.chunk",
239            "0124.primary",
240            "0124.secondary",
241            "223.chunk",
242            "223.primary",
243            "223.secondary",
244            "0423.chunk",
245            "0423.primary",
246            "0423.secondary",
247            "0424.chunk",
248            "0424.primary",
249            "0424.secondary",
250            "21.chunk",
251            "21.primary",
252            "21.secondary",
253        ];
254        create_fake_files(&target_dir, &entries);
255        let result = ImmutableFile::list_all_in_dir(target_dir.parent().unwrap())
256            .expect("ImmutableFile::list_in_dir Failed");
257
258        assert_eq!(result.last().unwrap().number, 424);
259        let expected_entries_length = 21;
260        assert_eq!(
261            expected_entries_length,
262            result.len(),
263            "Expected to find {} files but found {}",
264            entries.len(),
265            result.len(),
266        );
267    }
268
269    #[test]
270    fn list_completed_immutable_file_should_skip_last_number() {
271        let target_dir = get_test_dir("list_immutable_file_should_skip_last_number/immutable");
272        let entries = vec![
273            "123.chunk",
274            "123.primary",
275            "123.secondary",
276            "125.chunk",
277            "125.primary",
278            "125.secondary",
279            "0124.chunk",
280            "0124.primary",
281            "0124.secondary",
282            "223.chunk",
283            "223.primary",
284            "223.secondary",
285            "0423.chunk",
286            "0423.primary",
287            "0423.secondary",
288            "0424.chunk",
289            "0424.primary",
290            "0424.secondary",
291            "21.chunk",
292            "21.primary",
293            "21.secondary",
294        ];
295        create_fake_files(&target_dir, &entries);
296        let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
297            .expect("ImmutableFile::list_in_dir Failed");
298
299        assert_eq!(result.last().unwrap().number, 423);
300        assert_eq!(
301            result.len(),
302            entries.len() - 3,
303            "Expected to find {} files since the last (chunk, primary, secondary) trio is skipped, but found {}",
304            entries.len() - 3,
305            result.len(),
306        );
307    }
308
309    #[test]
310    fn list_completed_immutable_file_should_works_in_a_empty_folder() {
311        let target_dir =
312            get_test_dir("list_immutable_file_should_works_even_in_a_empty_folder/immutable");
313        let entries = vec![];
314        create_fake_files(&target_dir, &entries);
315        let result = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
316            .expect("ImmutableFile::list_in_dir Failed");
317
318        assert!(result.is_empty());
319    }
320
321    #[test]
322    fn list_completed_immutable_file_order_should_be_deterministic() {
323        let target_dir =
324            get_test_dir("list_completed_immutable_file_order_should_be_deterministic/immutable");
325        let entries = vec![
326            "21.chunk",
327            "21.primary",
328            "21.secondary",
329            "123.chunk",
330            "123.primary",
331            "123.secondary",
332            "124.chunk",
333            "124.primary",
334            "124.secondary",
335            "125.chunk",
336            "125.primary",
337            "125.secondary",
338            "223.chunk",
339            "223.primary",
340            "223.secondary",
341            "423.chunk",
342            "423.primary",
343            "423.secondary",
344            "424.chunk",
345            "424.primary",
346            "424.secondary",
347        ];
348        create_fake_files(&target_dir, &entries);
349        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
350            .expect("ImmutableFile::list_in_dir Failed");
351        let immutables_names: Vec<String> = extract_filenames(&immutables);
352
353        let expected: Vec<&str> = entries.into_iter().rev().skip(3).rev().collect();
354        assert_eq!(expected, immutables_names);
355    }
356
357    #[test]
358    fn list_completed_immutable_file_should_work_with_non_immutable_files() {
359        let target_dir =
360            get_test_dir("list_immutable_file_should_work_with_non_immutable_files/immutable");
361        let entries = vec![
362            "123.chunk",
363            "123.primary",
364            "123.secondary",
365            "124.chunk",
366            "124.primary",
367            "124.secondary",
368            "README.md",
369            "124.secondary.back",
370        ];
371        create_fake_files(&target_dir, &entries);
372        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
373            .expect("ImmutableFile::list_in_dir Failed");
374        let immutables_names: Vec<String> = extract_filenames(&immutables);
375
376        let expected: Vec<&str> = entries.into_iter().rev().skip(5).rev().collect();
377        assert_eq!(expected, immutables_names);
378    }
379
380    #[test]
381    fn list_completed_immutable_file_can_list_incomplete_trio() {
382        let target_dir = get_test_dir("list_immutable_file_can_list_incomplete_trio/immutable");
383        let entries = vec![
384            "21.chunk",
385            "21.primary",
386            "21.secondary",
387            "123.chunk",
388            "123.secondary",
389            "124.chunk",
390            "124.primary",
391            "125.primary",
392            "125.secondary",
393            "223.chunk",
394            "224.primary",
395            "225.secondary",
396            "226.chunk",
397        ];
398        create_fake_files(&target_dir, &entries);
399        let immutables = ImmutableFile::list_completed_in_dir(target_dir.parent().unwrap())
400            .expect("ImmutableFile::list_in_dir Failed");
401        let immutables_names: Vec<String> = extract_filenames(&immutables);
402
403        let expected: Vec<&str> = entries.into_iter().rev().skip(1).rev().collect();
404        assert_eq!(expected, immutables_names);
405    }
406}
mithril_common/digesters/immutable_file.rs

mithril_common/digesters/
immutable_file.rs
