Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: set size for tar directory entries #6100

Merged
merged 4 commits into from
Oct 5, 2023
Merged

fix: set size for tar directory entries #6100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b32bcd6
prefactor to allow for directories
Oct 4, 2023
b392398
add directory to test case
Oct 4, 2023
f13316d
set size for directories
Oct 5, 2023
77de92b
update windows snapshot test
Oct 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions crates/turborepo-cache/src/async_cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,11 @@ mod tests {
.put(
repo_root_path.clone(),
hash.clone(),
test_case.files.iter().map(|f| f.path.clone()).collect(),
test_case
.files
.iter()
.map(|f| f.path().to_owned())
.collect(),
test_case.duration,
)
.await
Expand Down Expand Up @@ -281,7 +285,11 @@ mod tests {
.put(
repo_root_path.clone(),
hash.clone(),
test_case.files.iter().map(|f| f.path.clone()).collect(),
test_case
.files
.iter()
.map(|f| f.path().to_owned())
.collect(),
test_case.duration,
)
.await
Expand Down Expand Up @@ -358,7 +366,11 @@ mod tests {
.put(
repo_root_path.clone(),
hash.clone(),
test_case.files.iter().map(|f| f.path.clone()).collect(),
test_case
.files
.iter()
.map(|f| f.path().to_owned())
.collect(),
test_case.duration,
)
.await
Expand Down
12 changes: 9 additions & 3 deletions crates/turborepo-cache/src/cache_archive/create.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,7 @@ impl<'a> CacheWriter<'a> {
header.set_link_name(link)?;
header.set_entry_type(EntryType::Symlink);
} else if file_info.is_dir() {
header.set_size(0);
header.set_entry_type(EntryType::Directory);
} else if file_info.is_file() {
header.set_entry_type(EntryType::Regular);
Expand Down Expand Up @@ -246,6 +247,7 @@ mod tests {
"bf0b4bf722f8d845dce7627606ab8af30bb6454d7c0379219e4c036a484960fe78e3d98e29ca0bac9b69b858d446b89d2d691c524e2884389032be799b6699f6",
"4f1357753cceec5df1c8a36110ce256f3e8c5c1f62cab3283013b6266d6e97b3884711ccdd45462a4607bee7ac7a8e414d0acea4672a9f0306bcf364281edc2f",
None
; "create regular file"
)]
#[test_case(
vec![
Expand Down Expand Up @@ -274,6 +276,7 @@ mod tests {
"0ece16efdb0b7e2a087e622ed52f29f21a4c080d77c31c4ed940b57dcdcb1f60b910d15232c0a2747325c22dadbfd069f15de969626dc49746be2d4b9b22e239",
"2e8ad9651964faa76082306dc95bff86fa0db821681e7a8acb982244ce0a9375417e867c3a9cb82f70bc6f03c7fb085e402712d3e9f27b980d5a0c22e086f4e2",
None
; "create symlinks"
)]
#[test_case(
vec![
Expand All @@ -288,10 +291,11 @@ mod tests {
file_type: FileType::File,
},
],
"027346e0349f948c0a2e7e9badb67d27fcc8ff4d5eacff1e5dd6a09c23a54d6793bf7ef1f25c9ed6b8c74f49d86d7b87478b7a00e24ea72e2ed2cadc0286c761",
"027346e0349f948c0a2e7e9badb67d27fcc8ff4d5eacff1e5dd6a09c23a54d6793bf7ef1f25c9ed6b8c74f49d86d7b87478b7a00e24ea72e2ed2cadc0286c761",
"1a2b32fe2b252ec622e5a15af21b274d702faa623d09c6fc51a44e7562cc84ac8b8c368d98d284dfb6666680ee252b071d5fbff44564a952ebaa12fe6f389e68",
"973c16d7e8662d3483ff0679b5337d7b9ba2001dbe863604fc8cc60254305750616312b9f988112db918d50fd087d89444d43a64beb4e8102109c5c628510131",
"973c16d7e8662d3483ff0679b5337d7b9ba2001dbe863604fc8cc60254305750616312b9f988112db918d50fd087d89444d43a64beb4e8102109c5c628510131",
"b8d51875d79a3cd56938e2ca8c3cad8eed6c96f7c38e152669ddfa7d7a1c44f62e4c3a13b299a30a895156c62b07ddbc46fdbf07a01870b965050359c19a9e06",
None
; "create directory"
)]
#[test_case(
vec![
Expand All @@ -305,6 +309,7 @@ mod tests {
"c113763393a9fb498cc676e1fe4843206cda665afe2144829fe7434da9e81f0cf6d11386fa79877d3c514d108f9696740256af952b57d32216fbed2eb2fb049d",
"fe692a000551a60da6cc303a9552a16d7ed5c462e33153a96824e96596da6d642fc671448f06f34e9685a13fe5bbb4220f59db73a856626b8a0962916a8f5ea3",
None
; "create broken symlink"
)]
#[test_case(
vec![
Expand All @@ -318,6 +323,7 @@ mod tests {
"",
"",
Some("attempted to create unsupported file type")
; "create unsupported"
)]
fn test_create(
files: Vec<CreateFileDefinition>,
Expand Down
16 changes: 13 additions & 3 deletions crates/turborepo-cache/src/fs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,7 @@ mod test {
use anyhow::Result;
use futures::future::try_join_all;
use tempfile::tempdir;
use turbopath::AnchoredSystemPath;

use super::*;
use crate::test_cases::{get_test_cases, TestCase};
Expand All @@ -181,7 +182,11 @@ mod test {
.expect_err("Expected cache miss");
assert_matches!(expected_miss, CacheError::CacheMiss);

let files: Vec<_> = test_case.files.iter().map(|f| f.path.clone()).collect();
let files: Vec<_> = test_case
.files
.iter()
.map(|f| f.path().to_owned())
.collect();
cache.put(repo_root_path, test_case.hash, &files, test_case.duration)?;

let expected_hit = cache.exists(test_case.hash)?;
Expand All @@ -204,9 +209,14 @@ mod test {

assert_eq!(files.len(), test_case.files.len());
for (expected, actual) in test_case.files.iter().zip(files.iter()) {
assert_eq!(&expected.path, actual);
let actual: &AnchoredSystemPath = actual;
assert_eq!(expected.path(), actual);
let actual_file = repo_root_path.resolve(actual);
assert_eq!(expected.contents, actual_file.read_to_string()?);
if let Some(contents) = expected.contents() {
assert_eq!(contents, actual_file.read_to_string()?);
} else {
assert!(actual_file.exists());
}
}

Ok(())
Expand Down
10 changes: 7 additions & 3 deletions crates/turborepo-cache/src/http.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ mod test {

let cache = HTTPCache::new(api_client, &opts, repo_root_path.to_owned(), api_auth);

let anchored_files: Vec<_> = files.iter().map(|f| f.path.clone()).collect();
let anchored_files: Vec<_> = files.iter().map(|f| f.path().to_owned()).collect();
cache
.put(&repo_root_path, hash, &anchored_files, duration)
.await?;
Expand All @@ -248,9 +248,13 @@ mod test {
assert_eq!(cache_response.time_saved, duration);

for (test_file, received_file) in files.iter().zip(received_files) {
assert_eq!(received_file, test_file.path);
assert_eq!(&*received_file, test_file.path());
let file_path = repo_root_path.resolve(&received_file);
assert_eq!(std::fs::read_to_string(file_path)?, test_file.contents);
if let Some(contents) = test_file.contents() {
assert_eq!(std::fs::read_to_string(file_path)?, contents);
} else {
assert!(file_path.exists());
}
}

Ok(())
Expand Down
88 changes: 59 additions & 29 deletions crates/turborepo-cache/src/test_cases.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,48 @@
use anyhow::Result;
use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf};
use turbopath::{AbsoluteSystemPath, AnchoredSystemPath, AnchoredSystemPathBuf};

pub(crate) struct TestFile {
pub path: AnchoredSystemPathBuf,
pub contents: &'static str,
path: AnchoredSystemPathBuf,
contents: Option<&'static str>,
}

impl TestFile {
pub fn file(path: AnchoredSystemPathBuf, contents: &'static str) -> Self {
Self {
path,
contents: Some(contents),
}
}

pub fn directory(path: AnchoredSystemPathBuf) -> Self {
Self {
path,
contents: None,
}
}

pub fn create(&self, repo_root: &AbsoluteSystemPath) -> Result<()> {
let file_path = repo_root.resolve(&self.path);
std::fs::create_dir_all(file_path.parent().unwrap())?;
std::fs::write(file_path, self.contents)?;
match self.contents {
Some(contents) => {
std::fs::create_dir_all(file_path.parent().unwrap())?;
std::fs::write(file_path, contents)?;
}
None => {
std::fs::create_dir(&file_path)?;
}
}

Ok(())
}

pub fn path(&self) -> &AnchoredSystemPath {
&self.path
}

pub fn contents(&self) -> Option<&str> {
self.contents
}
}

pub(crate) struct TestCase {
Expand All @@ -35,41 +64,42 @@ impl TestCase {
pub(crate) fn get_test_cases() -> Vec<TestCase> {
vec![
TestCase {
files: vec![TestFile {
path: AnchoredSystemPathBuf::from_raw("package.json").unwrap(),
contents: "hello world",
}],
files: vec![TestFile::file(
AnchoredSystemPathBuf::from_raw("package.json").unwrap(),
"hello world",
)],
duration: 58,
hash: "Faces Places",
},
TestCase {
files: vec![
TestFile {
path: AnchoredSystemPathBuf::from_raw("package.json").unwrap(),
contents: "Days of Heaven",
},
TestFile {
path: AnchoredSystemPathBuf::from_raw("package-lock.json").unwrap(),
contents: "Badlands",
},
TestFile::file(
AnchoredSystemPathBuf::from_raw("package.json").unwrap(),
"Days of Heaven",
),
TestFile::file(
AnchoredSystemPathBuf::from_raw("package-lock.json").unwrap(),
"Badlands",
),
],
duration: 1284,
hash: "Cleo from 5 to 7",
},
TestCase {
files: vec![
TestFile {
path: AnchoredSystemPathBuf::from_raw("package.json").unwrap(),
contents: "Days of Heaven",
},
TestFile {
path: AnchoredSystemPathBuf::from_raw("package-lock.json").unwrap(),
contents: "Badlands",
},
TestFile {
path: AnchoredSystemPathBuf::from_raw("src/main.js").unwrap(),
contents: "Tree of Life",
},
TestFile::file(
AnchoredSystemPathBuf::from_raw("package.json").unwrap(),
"Days of Heaven",
),
TestFile::file(
AnchoredSystemPathBuf::from_raw("package-lock.json").unwrap(),
"Badlands",
),
TestFile::directory(AnchoredSystemPathBuf::from_raw("src").unwrap()),
TestFile::file(
AnchoredSystemPathBuf::from_raw("src/main.js").unwrap(),
"Tree of Life",
),
],
duration: 12845,
hash: "The Gleaners and I",
Expand Down
Loading