Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: read from the local store if available #185

Open
wants to merge 10 commits into
base: main
Choose a base branch
from
41 changes: 29 additions & 12 deletions crates/package-manager/src/install_package_by_snapshot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
use pacquet_tarball::{DownloadTarballToStore, TarballError};
use pipe_trait::Pipe;
use reqwest::Client;
use std::borrow::Cow;
use std::{borrow::Cow, ffi::OsString, io::ErrorKind};

/// This subroutine downloads a package tarball, extracts it, installs it to a virtual dir,
/// then creates the symlink layout for the package.
Expand Down Expand Up @@ -56,17 +56,34 @@
}
};

// TODO: skip when already exists in store?
let cas_paths = DownloadTarballToStore {
http_client,
store_dir: &config.store_dir,
package_integrity: integrity,
package_unpacked_size: None,
package_url: &tarball_url,
}
.run_without_mem_cache()
.await
.map_err(InstallPackageBySnapshotError::DownloadTarball)?;
let store_dir = &config.store_dir;
let cas_paths = match store_dir.read_index_file(integrity) {
Ok(index) => store_dir
.cas_file_paths_by_index(&index)
.map(|(entry_path, store_path)| (OsString::from(entry_path), store_path))
.collect(),
Err(error) => {
if error.io_error_kind() != Some(ErrorKind::NotFound) {
let path = error.file_path();
tracing::warn!(
target: "pacquet::read_store",
?error,
?path,
"Failed to read index from store",
);
}
DownloadTarballToStore {
http_client,
store_dir,
package_integrity: integrity,
package_unpacked_size: None,
package_url: &tarball_url,
}
.run_without_mem_cache()
.await
.map_err(InstallPackageBySnapshotError::DownloadTarball)?

Check warning on line 84 in crates/package-manager/src/install_package_by_snapshot.rs

View check run for this annotation

Codecov / codecov/patch

crates/package-manager/src/install_package_by_snapshot.rs#L59-L84

Added lines #L59 - L84 were not covered by tests
}
};

CreateVirtualDirBySnapshot {
virtual_store_dir: &config.virtual_store_dir,
Expand Down
29 changes: 26 additions & 3 deletions crates/store-dir/src/cas_file.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
use crate::{FileHash, StoreDir};
use crate::{FileHash, PackageFilesIndex, StoreDir};
use derive_more::{Display, Error};
use miette::Diagnostic;
use pacquet_fs::{ensure_file, file_mode::EXEC_MODE, EnsureFileError};
use pacquet_fs::{ensure_file, file_mode, EnsureFileError};
use sha2::{Digest, Sha512};
use ssri::{Algorithm, Integrity};
use std::path::PathBuf;

impl StoreDir {
Expand All @@ -12,6 +13,28 @@
let suffix = if executable { "-exec" } else { "" };
self.file_path_by_hex_str(&hex, suffix)
}

/// List maps from index entry to real or would-be file path in the store directory.
pub fn cas_file_paths_by_index<'a>(
&'a self,
index: &'a PackageFilesIndex,
) -> impl Iterator<Item = (&'a str, PathBuf)> + 'a {
index.files.iter().map(|(entry_path, info)| {
let entry_path = entry_path.as_str();
let (algorithm, hex) = info
.integrity
.parse::<Integrity>()
.expect("parse integrity") // TODO: parse integrity before this
.to_hex();
assert!(
matches!(algorithm, Algorithm::Sha512),
"Only Sha512 is supported. {algorithm} isn't",

Check warning on line 31 in crates/store-dir/src/cas_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/cas_file.rs#L18-L31

Added lines #L18 - L31 were not covered by tests
); // TODO: write a custom parser and remove this
let suffix = if file_mode::is_all_exec(info.mode) { "-exec" } else { "" };
let cas_path = self.file_path_by_hex_str(&hex, suffix);
(entry_path, cas_path)
})
}

Check warning on line 37 in crates/store-dir/src/cas_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/cas_file.rs#L33-L37

Added lines #L33 - L37 were not covered by tests
}

/// Error type of [`StoreDir::write_cas_file`].
Expand All @@ -29,7 +52,7 @@
) -> Result<(PathBuf, FileHash), WriteCasFileError> {
let file_hash = Sha512::digest(buffer);
let file_path = self.cas_file_path(file_hash, executable);
let mode = executable.then_some(EXEC_MODE);
let mode = executable.then_some(file_mode::EXEC_MODE);
ensure_file(&file_path, buffer, mode).map_err(WriteCasFileError::WriteFile)?;
Ok((file_path, file_hash))
}
Expand Down
58 changes: 57 additions & 1 deletion crates/store-dir/src/index_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@
use pacquet_fs::{ensure_file, EnsureFileError};
use serde::{Deserialize, Serialize};
use ssri::{Algorithm, Integrity};
use std::{collections::HashMap, path::PathBuf};
use std::{
collections::HashMap,
fs::File,
io::{self, ErrorKind},
path::{Path, PathBuf},
};

impl StoreDir {
/// Path to an index file of a tarball.
Expand Down Expand Up @@ -58,6 +63,57 @@
}
}

/// Error type of [`StoreDir::read_index_file`].
#[derive(Debug, Display, Error, Diagnostic)]

Check warning on line 67 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L67

Added line #L67 was not covered by tests
pub enum ReadIndexFileError {
#[display("Failed to open {file_path:?}: {error}")]
OpenFile {
file_path: PathBuf,
#[error(source)]
error: io::Error,
},
#[display("Failed to parse content of {file_path:?}: {error}")]
ParseFile {
file_path: PathBuf,
#[error(source)]
error: serde_json::Error,
},
}

impl StoreDir {
/// Read an index file from the store directory.
pub fn read_index_file(
&self,
integrity: &Integrity,
) -> Result<PackageFilesIndex, ReadIndexFileError> {
let file_path = self.index_file_path(integrity);
let file = match File::open(&file_path) {
Ok(file) => file,
Err(error) => return Err(ReadIndexFileError::OpenFile { file_path, error }),

Check warning on line 92 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L85-L92

Added lines #L85 - L92 were not covered by tests
};
match serde_json::from_reader(file) {
Ok(content) => Ok(content),
Err(error) => Err(ReadIndexFileError::ParseFile { file_path, error }),

Check warning on line 96 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L94-L96

Added lines #L94 - L96 were not covered by tests
}
}

Check warning on line 98 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L98

Added line #L98 was not covered by tests
}

impl ReadIndexFileError {
pub fn file_path(&self) -> &Path {
match self {
ReadIndexFileError::OpenFile { file_path, .. } => file_path,
ReadIndexFileError::ParseFile { file_path, .. } => file_path,

Check warning on line 105 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L102-L105

Added lines #L102 - L105 were not covered by tests
}
}

Check warning on line 107 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L107

Added line #L107 was not covered by tests

pub fn io_error_kind(&self) -> Option<ErrorKind> {
match self {
ReadIndexFileError::OpenFile { error, .. } => Some(error.kind()),
ReadIndexFileError::ParseFile { error, .. } => error.io_error_kind(),

Check warning on line 112 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L109-L112

Added lines #L109 - L112 were not covered by tests
}
}

Check warning on line 114 in crates/store-dir/src/index_file.rs

View check run for this annotation

Codecov / codecov/patch

crates/store-dir/src/index_file.rs#L114

Added line #L114 was not covered by tests
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down