forked from Normation/rudder
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
11 changed files
with
252 additions
and
77 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| // SPDX-License-Identifier: GPL-3.0-or-later WITH GPL-3.0-linking-source-exception | ||
| // SPDX-FileCopyrightText: 2019-2020 Normation SAS | ||
|
|
||
| use std::{ | ||
| path::Path, | ||
| str::{self, FromStr}, | ||
| sync::Arc, | ||
| time::{SystemTime, UNIX_EPOCH}, | ||
| }; | ||
|
|
||
| use anyhow::{anyhow, Error}; | ||
| use tokio::{ | ||
| fs::{read, read_dir, remove_file}, | ||
| time::interval, | ||
| }; | ||
| use tracing::{debug, error, span, Level}; | ||
|
|
||
| use crate::{ | ||
| configuration::main::{SharedFilesCleanupConfig, WatchedDirectory}, | ||
| data::shared_file::Metadata, | ||
| JobConfig, | ||
| }; | ||
|
|
||
| pub fn start(job_config: &Arc<JobConfig>) { | ||
| let span = span!(Level::TRACE, "shared_files"); | ||
| let _enter = span.enter(); | ||
|
|
||
| let root_path = job_config.cfg.shared_files.path.clone(); | ||
|
|
||
| tokio::spawn(cleanup(root_path, job_config.cfg.shared_files.cleanup)); | ||
| } | ||
|
|
||
| async fn expired(file: &Path) -> Result<bool, Error> { | ||
| let raw = read(file.with_extension("metadata")).await?; | ||
| let metadata = str::from_utf8(&raw)?; | ||
|
|
||
| let parsed = Metadata::from_str(metadata)?; | ||
| let expiration = parsed | ||
| .expires | ||
| .ok_or(anyhow!("Missing expires field in {:?}", file))?; | ||
|
|
||
| let now = SystemTime::now() | ||
| .duration_since(UNIX_EPOCH) | ||
| .expect("Time went backwards") | ||
| .as_secs(); | ||
|
|
||
| Ok(expiration < now as i64) | ||
| } | ||
|
|
||
| // special cleanup implementation as retention is based on shared files metadata content | ||
| pub async fn cleanup(path: WatchedDirectory, cfg: SharedFilesCleanupConfig) -> Result<(), Error> { | ||
| let mut timer = interval(cfg.frequency); | ||
|
|
||
| loop { | ||
| timer.tick().await; | ||
| debug!("cleaning shared-files in {:?}", path); | ||
|
|
||
| // First level: source node directory | ||
| let mut dirs = match read_dir(path.clone()).await { | ||
| Ok(f) => f, | ||
| Err(e) => { | ||
| error!("list file: {}", e); | ||
| continue; | ||
| } | ||
| }; | ||
| loop { | ||
| let entry = match dirs.next_entry().await { | ||
| Ok(Some(e)) if e.file_type().await?.is_dir() => e, | ||
| // Nothing to do | ||
| Ok(_) => break, | ||
| Err(e) => { | ||
| error!("entry error: {}", e); | ||
| continue; | ||
| } | ||
| }; | ||
|
|
||
| // Second level: destination directory | ||
| let mut dirs = match read_dir(entry.path().join("files")).await { | ||
| Ok(f) => f, | ||
| Err(e) => { | ||
| error!("list file: {}", e); | ||
| continue; | ||
| } | ||
| }; | ||
| loop { | ||
| let entry = match dirs.next_entry().await { | ||
| Ok(Some(e)) if e.file_type().await?.is_dir() => e, | ||
| // Nothing to do | ||
| Ok(_) => break, | ||
| Err(e) => { | ||
| error!("entry error: {}", e); | ||
| continue; | ||
| } | ||
| }; | ||
|
|
||
| // Third level: shared file | ||
| let mut files = match read_dir(entry.path()).await { | ||
| Ok(f) => f, | ||
| Err(e) => { | ||
| error!("list file: {}", e); | ||
| continue; | ||
| } | ||
| }; | ||
| loop { | ||
| let entry = match files.next_entry().await { | ||
| Ok(Some(e)) if Path::exists(&e.path().with_extension("metadata")) => e, | ||
| // Nothing to do | ||
| Ok(_) => break, | ||
| Err(e) => { | ||
| error!("entry error: {}", e); | ||
| continue; | ||
| } | ||
| }; | ||
|
|
||
| let shared_file = entry.path(); | ||
| match expired(&shared_file).await { | ||
| Ok(true) => { | ||
| debug!("removing expired shared-file: {:?}", shared_file); | ||
| remove_file(shared_file) | ||
| .await | ||
| .unwrap_or_else(|e| error!("removal error: {}", e)); | ||
| } | ||
| _ => break, | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[cfg(test)] | ||
| mod tests { | ||
| use super::*; | ||
|
|
||
| #[tokio::test] | ||
| async fn it_reads_expire_metadata() { | ||
| assert!(expired(Path::new("tests/api_shared_files/37817c4d-fbf7-4850-a985-50021f4e8f41/files/e745a140-40bc-4b86-b6dc-084488fc906b/file.metadata")).await.unwrap()); | ||
| assert!(!expired(Path::new("tests/api_shared_files/37817c4d-fbf7-4850-a985-50021f4e8f41/files/e745a140-40bc-4b86-b6dc-084488fc906b/file-future.metadata")).await.unwrap()); | ||
| } | ||
| } |
9 changes: 9 additions & 0 deletions
9
...f7-4850-a985-50021f4e8f41/files/e745a140-40bc-4b86-b6dc-084488fc906b/file-future.metadata
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| header=rudder-signature-v1 | ||
| algorithm=sha512 | ||
| digest=9ae39f50bbbd3a2e529a1be61e739b814b9470a0704ef8dd7d3b78d44c2813ad062004f42dab3214a4779f8b4e8a5935cc92ff0313845377c58db605a0aaa5cffbe05e5809d4d646249cbe1fdbc6c498a7bc533415cb32de1b1f782c2f6b453b737999c4f8a15875cfe6a2808668e8c17986e100395633f702e709c2d49b7aa6eca6503e98360d58ca08ba0e91455512966a72036eeaa8c53dd016a129b2c366578c43d16c59208bdc34b39c81a8c88c073fc44bf64375644f6260d27de400ce790bd0b43251ec4640e2b9c0e2916b5e2d08395eec19f3f24a5c6bd39e2a6685f540e43c43508a98bca5cb7825022db8169080d9d3f8660ffc9b11aaa9b8088311f9bc3a0f1d822055d7ec1fedb9adabeeac7625106fa0597dfebd0d22e4703feedc2137ff705193249707e228a139015f49455fa1742e83676b5af5f84e58b71bad7846fcd731326982a4f48fd301360cc6e911554cdf193b99c7491c707b2ea373bc7ebf71d02bcdd34f4812b701977e79e7ed1b9ab6522f5e8e048cf024f24e7f0a2fcc47d6f72543acf421560da8aa9bc71c88083173122fb4905e683f4be91155d8c87f948b8b0357b10594dd61cf33dff8f3bb418a1fdc5f88ea0589906adbff1774d310829576326e4d376a8d6bc495240d030e79c76bde2daa95c051aae8b8fb2dca9327b120d494d6e9fe4dcd30cda6ba6a110cc3055a6f2a4b146d | ||
| hash_value=dda78e9b97a69aca3cff21de266246bde0d91bc4b61df72bfb0387564ac0c7bd64dd4caca39ce1ef400f32aa711ec4909789705beec93314eb65fabd5183bbfe | ||
| short_pubkey=MIICCgKCAgEAuok8JTvRssiupO0IfH4OGnWFqQg5dmI/4JsCiPEUf78iFBwFFpwuNXDJXCKaHtpjuc3DAy9l7fmZ+bQmkfde+Qo3yAd2ZsId80TBZOy6uFQyl4ASLNgY8RKIFxD6+AsutI27KexSnL3QLCgywnheRv4Ur31a6MVY1xfSQnADruBBad+5SaF3hTpEcAMg2hDQsIcyR32MPRy9MOVmvBlgI2hZsgh9QQf9wTLxGuMw/pJKOPRwwFkk/5bhFBve2sL1OI0pRsM6i7SxNXRhM6NWlmObhP+Z7C6N7TY00Z+tizgETmYJ35llyInjc1i+0bWaj5p3cbSCVdQ5zomZ3L9XbsWmjl0P/cw06qqNPuLR799K+R1XgA94nUUzo2pVigPh6sj2XMS8FOWXMXy2TNEOA+NQV5+vYwIlUizvB/HHSc3WKqNGgCifdJBmJJ8QTg5cJE6s+91O99eMMAQ0Ecj+nY5QEYkbIn4gjNpojam3jyS72o0J4nlj4ECbR/rj6L5b+kj5F3DbYqSdLC+crKUIoBZH1msCuJcQ9Zk/YHw87iVyWoZOVtJUUaw3n8vH/YCWPBQRzZp+4zlyIYJIIz+V/FJZX5YNW9XgoeRG8Q0mOmLy0FbQUS/klYlpeW3PKLSQmcSLvrgZnhKMyhEohC0zOSqJU0ui4VUWY5tv1bhbTo8CAwEAAQ== | ||
| hostname=node1.rudder.local | ||
| keydate=2020-01-24 12:17:59.014153459 +0100 | ||
| keyid=B85B4E8F | ||
| expires=2061475500 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.