Skip to content

Commit

Permalink
Merge 572015c into 2a4a1eb
Browse files Browse the repository at this point in the history
  • Loading branch information
@pkhuong
pkhuong committed Sep 6, 2021
2 parents 2a4a1eb + 572015c commit 0eb18a0
Show file tree
Hide file tree
Showing 5 changed files with 750 additions and 82 deletions.
92 changes: 76 additions & 16 deletions src/cache_dir.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
use std::borrow::Cow;
use std::fs::File;
use std::io::ErrorKind;
use std::io::Result;
use std::path::Path;
use std::path::PathBuf;
use std::time::Duration;

use crate::raw_cache;
Expand All @@ -15,16 +17,36 @@ const MAX_TEMP_FILE_AGE: Duration = Duration::from_secs(3600);
#[cfg(test)]
const MAX_TEMP_FILE_AGE: Duration = Duration::from_secs(2);

/// Attempts to make sure `path` is a directory that exists. Unlike
/// `std::fs::create_dir_all`, this function is optimised for the case
/// where `path` is already a directory.
fn ensure_directory(path: &Path) -> Result<()> {
if let Ok(meta) = std::fs::metadata(path) {
if meta.file_type().is_dir() {
return Ok(());
}
}

std::fs::create_dir_all(path)
}

/// Deletes any file with mtime older than `MAX_TEMP_FILE_AGE` in
/// `temp_dir`.
///
/// It is not an error if `temp_dir` does not exist.
fn cleanup_temporary_directory(temp_dir: Cow<Path>) -> Result<()> {
let threshold = match std::time::SystemTime::now().checked_sub(MAX_TEMP_FILE_AGE) {
Some(time) => time,
None => return Ok(()),
};

let iter = match std::fs::read_dir(&temp_dir) {
Err(e) if e.kind() == ErrorKind::NotFound => return Ok(()),
x => x?,
};

let mut temp = temp_dir.into_owned();
for dirent in std::fs::read_dir(&temp)?.flatten() {
for dirent in iter.flatten() {
let mut handle = || -> Result<()> {
let metadata = dirent.metadata()?;
let mtime = metadata.modified()?;
Expand Down Expand Up @@ -59,6 +81,14 @@ pub(crate) trait CacheDir {
/// Returns the cache's directory capacity (in object count).
fn capacity(&self) -> usize;

/// Return the path for the cache directory's temporary
/// subdirectory, after making sure the directory exists.
fn ensure_temp_dir(&self) -> Result<Cow<Path>> {
let ret = self.temp_dir();
ensure_directory(&ret)?;
Ok(ret)
}

/// Returns a read-only file for `name` in the cache directory if
/// it exists, or None if there is no such file.
///
Expand All @@ -80,52 +110,82 @@ pub(crate) trait CacheDir {
cleanup_temporary_directory(self.temp_dir())
}

/// Updates the second chance cache state and deletes temporary
/// files in the `base_dir` cache directory.
fn definitely_cleanup(&self, base_dir: PathBuf) -> Result<u64> {
let ret = match raw_cache::prune(base_dir, self.capacity()) {
Ok((estimate, _deleted)) => estimate,
Err(e) if e.kind() == ErrorKind::NotFound => return Ok(0),
Err(e) => return Err(e),
};

// Delete old temporary files while we're here.
self.cleanup_temp_directory()?;
Ok(ret)
}

/// If a periodic cleanup is called for, updates the second chance
/// cache state and deletes temporary files in that cache directory.
///
/// Returns true whenever cleanup was initiated.
fn maybe_cleanup(&self, base_dir: &Path) -> Result<bool> {
/// Returns the estimated number of files remaining after cleanup
/// whenever cleanup was initiated.
fn maybe_cleanup(&self, base_dir: &Path) -> Result<Option<u64>> {
if self.trigger().event() {
raw_cache::prune(base_dir.to_owned(), self.capacity())?;
// Delete old temporary files while we're here.
self.cleanup_temp_directory()?;
Ok(true)
Ok(Some(self.definitely_cleanup(base_dir.to_owned())?))
} else {
Ok(false)
Ok(None)
}
}

/// Updates the second chance cache state and deletes temporary
/// files in the `base_dir` cache directory.
///
/// Returns the estimated number of files remaining after cleanup.
fn maintain(&self) -> Result<u64> {
self.definitely_cleanup(self.base_dir().into_owned())
}

/// Inserts or overwrites the file at `value` as `name` in the
/// cache directory.
///
/// Returns the estimated number of files remaining after cleanup
/// whenever cleanup was initiated.
///
/// Always consumes the file at `value` on success; may consume it
/// on error.
fn set(&self, name: &str, value: &Path) -> Result<()> {
fn set(&self, name: &str, value: &Path) -> Result<Option<u64>> {
let mut dst = self.base_dir().into_owned();

self.maybe_cleanup(&dst)?;
let ret = self.maybe_cleanup(&dst)?;
ensure_directory(&dst)?;
dst.push(name);
raw_cache::insert_or_update(value, &dst)
raw_cache::insert_or_update(value, &dst)?;
Ok(ret)
}

/// Inserts the file at `value` as `name` in the cache directory
/// if there is no such cached entry already, or touches the
/// cached file if it already exists.
///
/// Returns the estimated number of files remaining after cleanup
/// whenever cleanup was initiated.
///
/// Always consumes the file at `value` on success; may consume it
/// on error.
fn put(&self, name: &str, value: &Path) -> Result<()> {
fn put(&self, name: &str, value: &Path) -> Result<Option<u64>> {
let mut dst = self.base_dir().into_owned();

self.maybe_cleanup(&dst)?;
let ret = self.maybe_cleanup(&dst)?;
ensure_directory(&dst)?;
dst.push(name);
raw_cache::insert_or_touch(value, &dst)
raw_cache::insert_or_touch(value, &dst)?;
Ok(ret)
}

/// Marks the cached file `name` as newly used, if it exists.
///
/// Succeeds if `name` does not exist anymore.
fn touch(&self, name: &str) -> Result<()> {
/// Returns whether the file `name` exists.
fn touch(&self, name: &str) -> Result<bool> {
let mut target = self.base_dir().into_owned();
target.push(name);

Expand Down
3 changes: 3 additions & 0 deletions src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ mod cache_dir;
mod plain;
pub mod raw_cache;
pub mod second_chance;
mod sharded;
mod trigger;

pub use plain::PlainCache;
pub use sharded::Key;
pub use sharded::ShardedCache;
69 changes: 37 additions & 32 deletions src/plain.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,22 +61,15 @@ impl CacheDir for PlainCache {
impl PlainCache {
/// Returns a new cache for approximately `capacity` files in
/// `base_dir`.
///
/// # Errors
///
/// Returns `Err` if `$base_dir/.temp` does not exist and we fail
/// to create it.
pub fn new(base_dir: PathBuf, capacity: usize) -> Result<PlainCache> {
pub fn new(base_dir: PathBuf, capacity: usize) -> PlainCache {
let mut temp_dir = base_dir;

temp_dir.push(TEMP_SUBDIR);
std::fs::create_dir_all(&temp_dir)?;

Ok(PlainCache {
PlainCache {
temp_dir,
trigger: PeriodicTrigger::new((capacity / MAINTENANCE_SCALE) as u64),
capacity,
})
}
}

/// Returns a read-only file for `name` in the cache directory if
Expand All @@ -89,8 +82,8 @@ impl PlainCache {

/// Returns a temporary directory suitable for temporary files
/// that will be published to the cache directory.
pub fn temp_dir(&self) -> Cow<Path> {
CacheDir::temp_dir(self)
pub fn temp_dir(&self) -> Result<Cow<Path>> {
CacheDir::ensure_temp_dir(self)
}

/// Inserts or overwrites the file at `value` as `name` in the
Expand All @@ -99,7 +92,8 @@ impl PlainCache {
/// Always consumes the file at `value` on success; may consume it
/// on error.
pub fn set(&self, name: &str, value: &Path) -> Result<()> {
CacheDir::set(self, name, value)
CacheDir::set(self, name, value)?;
Ok(())
}

/// Inserts the file at `value` as `name` in the cache directory
Expand All @@ -109,13 +103,14 @@ impl PlainCache {
/// Always consumes the file at `value` on success; may consume it
/// on error.
pub fn put(&self, name: &str, value: &Path) -> Result<()> {
CacheDir::put(self, name, value)
CacheDir::put(self, name, value)?;
Ok(())
}

/// Marks the cached file `name` as newly used, if it exists.
///
/// Succeeds even if `name` does not exist.
pub fn touch(&self, name: &str) -> Result<()> {
/// Returns whether `name` exists.
pub fn touch(&self, name: &str) -> Result<bool> {
CacheDir::touch(self, name)
}
}
Expand All @@ -140,12 +135,13 @@ fn smoke_test() {

// Make sure the garbage file is old enough to be deleted.
std::thread::sleep(std::time::Duration::from_secs_f64(2.5));
let cache = PlainCache::new(temp.path("."), 10).expect("::new must succeed");
let cache = PlainCache::new(temp.path("."), 10);

for i in 0..20 {
let name = format!("{}", i);

let tmp = NamedTempFile::new_in(cache.temp_dir()).expect("new temp file must succeed");
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
std::fs::write(tmp.path(), format!("{}", PAYLOAD_MULTIPLIER * i))
.expect("write must succeed");
cache.put(&name, tmp.path()).expect("put must succeed");
Expand Down Expand Up @@ -185,10 +181,11 @@ fn test_set() {
use test_dir::{DirBuilder, TestDir};

let temp = TestDir::temp();
let cache = PlainCache::new(temp.path("."), 1).expect("::new must succeed");
let cache = PlainCache::new(temp.path("."), 1);

{
let tmp = NamedTempFile::new_in(cache.temp_dir()).expect("new temp file must succeed");
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp.as_file().write_all(b"v1").expect("write must succeed");

cache
Expand All @@ -208,7 +205,8 @@ fn test_set() {

// Now overwrite; it should take.
{
let tmp = NamedTempFile::new_in(cache.temp_dir()).expect("new temp file must succeed");
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp.as_file().write_all(b"v2").expect("write must succeed");

cache
Expand Down Expand Up @@ -236,10 +234,11 @@ fn test_put() {
use test_dir::{DirBuilder, TestDir};

let temp = TestDir::temp();
let cache = PlainCache::new(temp.path("."), 1).expect("::new must succeed");
let cache = PlainCache::new(temp.path("."), 1);

{
let tmp = NamedTempFile::new_in(cache.temp_dir()).expect("new temp file must succeed");
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp.as_file().write_all(b"v1").expect("write must succeed");

cache
Expand All @@ -259,7 +258,8 @@ fn test_put() {

// Now put again; it shouldn't overwrite.
{
let tmp = NamedTempFile::new_in(cache.temp_dir()).expect("new temp file must succeed");
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
tmp.as_file().write_all(b"v2").expect("write must succeed");

cache
Expand All @@ -286,18 +286,22 @@ fn test_touch() {
use test_dir::{DirBuilder, TestDir};

let temp = TestDir::temp();
let cache = PlainCache::new(temp.path("."), 5).expect("::new must succeed");
let cache = PlainCache::new(temp.path("."), 5);

for i in 0..15 {
let name = format!("{}", i);

cache.touch("0").expect("touch must not fail");
// After the first write, touch should find our file.
assert_eq!(cache.touch("0").expect("touch must not fail"), i > 0);

let tmp = NamedTempFile::new_in(cache.temp_dir()).expect("new temp file must succeed");
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
cache.put(&name, tmp.path()).expect("put must succeed");
// Make sure enough time elapses for the next file to get
// a different timestamp.
std::thread::sleep(std::time::Duration::from_secs_f64(1.5));
// Make sure enough time elapses for the first file to get
// an older timestamp than the rest.
if i == 0 {
std::thread::sleep(std::time::Duration::from_secs_f64(1.5));
}
}

// We should still find "0": it's the oldest, but we also keep
Expand All @@ -320,10 +324,11 @@ fn test_recent_temp_file() {
// The garbage file must exist.
assert!(std::fs::metadata(temp.path(&format!("{}/garbage", TEMP_SUBDIR))).is_ok());

let cache = PlainCache::new(temp.path("."), 1).expect("::new must succeed");
let cache = PlainCache::new(temp.path("."), 1);

for i in 0..2 {
let tmp = NamedTempFile::new_in(cache.temp_dir()).expect("new temp file must succeed");
let tmp = NamedTempFile::new_in(cache.temp_dir().expect("temp_dir must succeed"))
.expect("new temp file must succeed");
cache
.put(&format!("{}", i), tmp.path())
.expect("put must succeed");
Expand Down
Loading

0 comments on commit 0eb18a0

Please sign in to comment.