Skip to content

Commit

Permalink
implement Step::CacheCheck
Browse files Browse the repository at this point in the history
  • Loading branch information
@TheAlan404
TheAlan404 committed Jun 19, 2024
1 parent 65219b7 commit d4abb48
Show file tree
Hide file tree
Showing 2 changed files with 131 additions and 16 deletions.
134 changes: 128 additions & 6 deletions src/api/app/step.rs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
use std::path::Path;
use std::{os::unix::fs::MetadataExt, path::Path};

use anyhow::{anyhow, bail, Result};
use tokio::{fs::File, io::BufWriter};
use tokio_stream::StreamExt;
use tokio_util::io::ReaderStream;

use crate::api::{
models::Addon,

Check warning on line 9 in src/api/app/step.rs

View workflow job for this annotation

GitHub Actions / clippy

unused imports: `CacheLocation`, `FileMeta`, `models::Addon`, `utils::hashing::get_best_hash` 

warning: unused imports: `CacheLocation`, `FileMeta`, `models::Addon`, `utils::hashing::get_best_hash`
  --> src/api/app/step.rs:9:5
   |
9  |     models::Addon,
   |     ^^^^^^^^^^^^^
10 |     step::{CacheLocation, FileMeta, Step, StepResult},
   |            ^^^^^^^^^^^^^  ^^^^^^^^
11 |     utils::hashing::get_best_hash,
   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
step::{CacheLocation, Step, StepResult},
step::{CacheLocation, FileMeta, Step, StepResult},
utils::hashing::get_best_hash,
};

Expand All @@ -26,12 +28,133 @@ impl App {

pub async fn execute_step(&self, dir: &Path, step: &Step) -> Result<StepResult> {
match step {
// cache | output | to do
// x | x | StepResult::Skip
// x | | copy from cache
// | x | StepResult::Continue
// | | StepResult::Continue
Step::CacheCheck(metadata) => {
if let Some(cache) = &metadata.cache {}
let output_path = dir.join(&metadata.filename);

Ok(StepResult::Continue)
let Some(cached_path) = self.cache.loc(metadata.cache.as_ref()) else {
return Ok(StepResult::Continue);
};

if !cached_path.try_exists()? {
return Ok(StepResult::Continue);
}

let cached_meta = cached_path.metadata()?;
let cached_size = cached_meta.size();

let output_size = if output_path.try_exists()? {
let meta = output_path.metadata()?;
Some(meta.size())
} else {
None
};

if let Some(output_size) = output_size {
// if output file exists...

if output_size != cached_size || metadata.size.is_some_and(|x| x != output_size) {
// size mismatch
// TODO: print warning
println!("WARNING size mismatch: {}", metadata.filename);
tokio::fs::remove_file(&output_path).await?;
//return Ok(StepResult::Continue);
} else {
if let Some((format, mut hasher, content)) = metadata.get_hasher() {

Check warning on line 67 in src/api/app/step.rs

View workflow job for this annotation

GitHub Actions / clippy

unused variable: `format` 

warning: unused variable: `format`
  --> src/api/app/step.rs:67:38
   |
67 |                         if let Some((format, mut hasher, content)) = metadata.get_hasher() {
   |                                      ^^^^^^ help: if this is intentional, prefix it with an underscore: `_format`
let output_file = File::open(&output_path).await?;
let mut stream = ReaderStream::new(output_file);

while let Some(item) = stream.try_next().await? {
hasher.update(&item);
}

let hash = hex::encode(&hasher.finalize());

if content == hash {
// size and hash match, skip rest of the steps
// TODO: print info
println!("Skipping (output hash matches) {}", metadata.filename);
return Ok(StepResult::Skip);
} else {
// hash mismatch
// TODO: print warning
println!("WARNING Hash mismatch: {}", metadata.filename);
tokio::fs::remove_file(&output_path).await?;
}

Check warning on line 87 in src/api/app/step.rs

View workflow job for this annotation

GitHub Actions / clippy

redundant else block 

warning: redundant else block
  --> src/api/app/step.rs:82:36
   |
82 |   ...                   } else {
   |  ______________________________^
83 | | ...                       // hash mismatch
84 | | ...                       // TODO: print warning
85 | | ...                       println!("WARNING Hash mismatch: {}", metadata.filename);
86 | | ...                       tokio::fs::remove_file(&output_path).await?;
87 | | ...                   }
   | |_______________________^
   |
   = help: remove the `else` block and move the contents out
   = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#redundant_else
   = note: `-W clippy::redundant-else` implied by `-W clippy::pedantic`
   = help: to override `-W clippy::pedantic` add `#[allow(clippy::redundant_else)]`
} else {
// FileInfo doesn't have any hashes
// so we must check from cache
// return skip if equal, do nothing otherwise to fallback copyfromcache
let target_file = File::open(&output_path).await?;
let cached_file = File::open(&cached_path).await?;

let mut target_stream = ReaderStream::new(target_file);
let mut cached_stream = ReaderStream::new(cached_file);

let is_equal = loop {
match (target_stream.next().await, cached_stream.next().await) {
(Some(Ok(a)), Some(Ok(b))) => {
if a != b {
break false;
}
},
(None, None) => break true,
_ => break false,
}
};

if is_equal {
// TODO: print info
println!("Skipping (eq cached) {}", metadata.filename);
return Ok(StepResult::Skip);
}
}
}

Check failure on line 116 in src/api/app/step.rs

View workflow job for this annotation

GitHub Actions / clippy

this `else { if .. }` block can be collapsed 

error: this `else { if .. }` block can be collapsed
   --> src/api/app/step.rs:66:28
    |
66  |                       } else {
    |  ____________________________^
67  | |                         if let Some((format, mut hasher, content)) = metadata.get_hasher() {
68  | |                             let output_file = File::open(&output_path).await?;
69  | |                             let mut stream = ReaderStream::new(output_file);
...   |
115 | |                         }
116 | |                     }
    | |_____________________^
    |
    = help: for further information visit https://rust-lang.github.io/rust-clippy/master/index.html#collapsible_else_if
    = note: `-D clippy::collapsible-else-if` implied by `-D clippy::all`
    = help: to override `-D clippy::all` add `#[allow(clippy::collapsible_else_if)]`
help: collapse nested if block
    |
66  ~                     } else if let Some((format, mut hasher, content)) = metadata.get_hasher() {
67  +                                             let output_file = File::open(&output_path).await?;
68  +                                             let mut stream = ReaderStream::new(output_file);
69  +                     
70  +                                             while let Some(item) = stream.try_next().await? {
71  +                                                 hasher.update(&item);
72  +                                             }
73  +                     
74  +                                             let hash = hex::encode(&hasher.finalize());
75  +                                             
76  +                                             if content == hash {
77  +                                                 // size and hash match, skip rest of the steps
78  +                                                 // TODO: print info
79  +                                                 println!("Skipping (output hash matches) {}", metadata.filename);
80  +                                                 return Ok(StepResult::Skip);
81  +                                             } else {
82  +                                                 // hash mismatch
83  +                                                 // TODO: print warning
84  +                                                 println!("WARNING Hash mismatch: {}", metadata.filename);
85  +                                                 tokio::fs::remove_file(&output_path).await?;
86  +                                             }
87  +                                         } else {
88  +                                             // FileInfo doesn't have any hashes
89  +                                             // so we must check from cache
90  +                                             // return skip if equal, do nothing otherwise to fallback copyfromcache
91  +                                             let target_file = File::open(&output_path).await?;
92  +                                             let cached_file = File::open(&cached_path).await?;
93  +                     
94  +                                             let mut target_stream = ReaderStream::new(target_file);
95  +                                             let mut cached_stream = ReaderStream::new(cached_file);
96  +                     
97  +                                             let is_equal = loop {
98  +                                                 match (target_stream.next().await, cached_stream.next().await) {
99  +                                                     (Some(Ok(a)), Some(Ok(b))) => {
100 +                                                         if a != b {
101 +                                                             break false;
102 +                                                         }
103 +                                                     },
104 +                                                     (None, None) => break true,
105 +                                                     _ => break false,
106 +                                                 }
107 +                                             };
108 +                     
109 +                                             if is_equal {
110 +                                                 // TODO: print info
111 +                                                 println!("Skipping (eq cached) {}", metadata.filename);
112 +                                                 return Ok(StepResult::Skip);
113 +                                             }
114 +                                         }
    |
}

// == Copying from cache ==

let mut hasher = metadata.get_hasher();

let target_file = File::create(&output_path).await?;
let mut target_writer = BufWriter::new(target_file);

let cached_file = File::open(&cached_path).await?;
let mut stream = ReaderStream::new(cached_file);

while let Some(item) = stream.try_next().await? {
if let Some((_, ref mut digest, _)) = hasher {
digest.update(&item);
}

tokio::io::copy(&mut item.as_ref(), &mut target_writer).await?;
}

if let Some((_, hasher, content)) = hasher {
let hash = hex::encode(&hasher.finalize());

if hash != content {
// TODO: print warning
println!("WARNING Hash Mismatch on CacheCopy: {}", metadata.filename);
tokio::fs::remove_file(&output_path).await?;
tokio::fs::remove_file(&cached_path).await?;
return Ok(StepResult::Continue);
}
}

println!("Copied: {}", metadata.filename);
Ok(StepResult::Skip)
}

// if FileMeta has cache location,
// download to cache dir
// copy from cache
// else
// download to destination dir
Step::Download { url, metadata } => {
let cache_destination = self.cache.loc(metadata.cache.as_ref());
let output_destination = dir.join(&metadata.filename);
Expand All @@ -48,8 +171,7 @@ impl App {

let mut stream = res.bytes_stream();

let mut hasher = get_best_hash(&metadata.hashes)
.map(|(format, content)| (format, format.get_digest(), content));
let mut hasher = metadata.get_hasher();

let target_destination = cache_destination.as_ref().unwrap_or(&output_destination);
tokio::fs::create_dir_all(target_destination.parent().ok_or(anyhow!("No parent"))?)
Expand Down
13 changes: 3 additions & 10 deletions src/api/step/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
use std::{borrow::Cow, collections::HashMap, path::Path};

Check warning on line 1 in src/api/step/mod.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `path::Path` 

warning: unused import: `path::Path`
 --> src/api/step/mod.rs:1:46
  |
1 | use std::{borrow::Cow, collections::HashMap, path::Path};
  |                                              ^^^^^^^^^^

use anyhow::Result;

Check warning on line 3 in src/api/step/mod.rs

View workflow job for this annotation

GitHub Actions / clippy

unused import: `anyhow::Result` 

warning: unused import: `anyhow::Result`
 --> src/api/step/mod.rs:3:5
  |
3 | use anyhow::Result;
  |     ^^^^^^^^^^^^^^
use digest::DynDigest;
use serde::{Deserialize, Serialize};

use crate::api::utils::hashing::HashFormat;
Expand All @@ -27,16 +28,8 @@ pub struct FileMeta {
}

impl FileMeta {
pub fn does_match<P: AsRef<Path>>(&self, file: P) -> Result<bool> {
let metadata = std::fs::metadata(file.as_ref())?;

if self.size.is_some_and(|size| metadata.len() != size) {
return Ok(false);
}

if let Some((format, hash)) = get_best_hash(&self.hashes) {}

Ok(true)
pub fn get_hasher(&self) -> Option<(HashFormat, Box<dyn DynDigest>, String)> {
get_best_hash(&self.hashes).map(|(format, content)| (format, format.get_digest(), content))
}
}

Expand Down

0 comments on commit d4abb48

Please sign in to comment.