diff --git a/relay/sources/Makefile b/relay/sources/Makefile index dd5ac46bdbf..5273e4320fa 100644 --- a/relay/sources/Makefile +++ b/relay/sources/Makefile @@ -90,7 +90,6 @@ install: build mkdir -p $(DESTDIR)/var/rudder/share mkdir -p $(DESTDIR)/var/log/rudder/apache2/ mkdir -p $(DESTDIR)/etc/sysconfig/ - mkdir -p $(DESTDIR)/etc/cron.d/ mkdir -p $(DESTDIR)/etc/sudoers.d/ mkdir -p $(DESTDIR)/usr/lib/systemd/system/ @@ -126,9 +125,7 @@ install: build # Others install -m 644 openssl.cnf $(DESTDIR)/opt/rudder/etc/ssl/openssl.cnf install -m 644 rudder-relay-apache $(DESTDIR)/etc/sysconfig/rudder-relay-apache - install -m 644 rudder-relay.cron $(DESTDIR)/etc/cron.d/rudder-relay install -m 644 rudder-relay.sudo $(DESTDIR)/etc/sudoers.d/rudder-relay - install -m 755 relay-cleanup $(DESTDIR)/opt/rudder/bin/relay-cleanup # Copy stub rudder-networks*.conf install -m 644 apache/rudder-networks-24.conf $(DESTDIR)/opt/rudder/etc/ diff --git a/relay/sources/relay-cleanup b/relay/sources/relay-cleanup deleted file mode 100755 index 8253e31f90c..00000000000 --- a/relay/sources/relay-cleanup +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -# remove all files and their metadata in BASEDIR that have expired - -BASEDIR="/var/rudder/shared-files" -date=$(date +%s) -find "${BASEDIR}" -type f -name '*.metadata' | xargs grep -H 'expires=' | sed 's/^\(.*\).metadata:expires=/\1 /' | -while read f d -do - if [ ${date} -gt ${d} ] - then - rm "${f}" "${f}.metadata" - fi -done diff --git a/relay/sources/relayd/src/configuration/main.rs b/relay/sources/relayd/src/configuration/main.rs index d62a48e2bee..96e34aeeb5c 100644 --- a/relay/sources/relayd/src/configuration/main.rs +++ b/relay/sources/relayd/src/configuration/main.rs @@ -1,12 +1,6 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH GPL-3.0-linking-source-exception // SPDX-FileCopyrightText: 2019-2020 Normation SAS -use crate::{configuration::Secret, data::node::NodeId}; -use anyhow::{anyhow, Error}; -use serde::{ - de::{Deserializer, Error as SerdeError, Unexpected, Visitor}, - Deserialize, -}; use std::{ collections::HashSet, convert::TryFrom, @@ -16,8 +10,16 @@ use std::{ str::FromStr, time::Duration, }; + +use anyhow::{anyhow, Error}; +use serde::{ + de::{Deserializer, Error as SerdeError, Unexpected, Visitor}, + Deserialize, +}; use tracing::{debug, warn}; +use crate::{configuration::Secret, data::node::NodeId}; + pub type BaseDirectory = PathBuf; pub type WatchedDirectory = PathBuf; pub type NodesListFile = PathBuf; @@ -472,10 +474,34 @@ impl Default for RemoteRun { } } +#[derive(Deserialize, Debug, PartialEq, Eq, Copy, Clone)] +pub struct SharedFilesCleanupConfig { + #[serde(deserialize_with = "compat_humantime")] + #[serde(default = "CleanupConfig::default_cleanup_frequency")] + pub frequency: Duration, +} + +impl SharedFilesCleanupConfig { + /// 10 minutes + fn default_cleanup_frequency() -> Duration { + Duration::from_secs(600) + } +} + +impl Default for SharedFilesCleanupConfig { + fn default() -> Self { + Self { + frequency: Self::default_cleanup_frequency(), + } + } +} + #[derive(Deserialize, Debug, PartialEq, Eq, Clone)] pub struct SharedFiles { #[serde(default = "SharedFiles::default_path")] pub path: PathBuf, + #[serde(default)] + pub cleanup: SharedFilesCleanupConfig, } impl SharedFiles { @@ -488,6 +514,7 @@ impl Default for SharedFiles { fn default() -> Self { Self { path: Self::default_path(), + cleanup: SharedFilesCleanupConfig::default(), } } } @@ -616,9 +643,10 @@ impl Default for UpstreamConfig { #[cfg(test)] mod tests { - use super::*; use pretty_assertions::assert_eq; + use super::*; + #[test] fn it_parses_listen_with_hostname() { let default = "[general]\n\ @@ -715,6 +743,9 @@ mod tests { }, shared_files: SharedFiles { path: PathBuf::from("/var/rudder/shared-files/"), + cleanup: SharedFilesCleanupConfig { + frequency: Duration::from_secs(600), + }, }, shared_folder: SharedFolder { path: PathBuf::from("/var/rudder/configuration-repository/shared-files/"), @@ -813,6 +844,9 @@ mod tests { }, shared_files: SharedFiles { path: PathBuf::from("tests/api_shared_files"), + cleanup: SharedFilesCleanupConfig { + frequency: Duration::from_secs(43), + }, }, shared_folder: SharedFolder { path: PathBuf::from("tests/api_shared_folder"), diff --git a/relay/sources/relayd/src/processing.rs b/relay/sources/relayd/src/processing.rs index cafd88d6fc6..87f6b164d1e 100644 --- a/relay/sources/relayd/src/processing.rs +++ b/relay/sources/relayd/src/processing.rs @@ -1,13 +1,15 @@ // SPDX-License-Identifier: GPL-3.0-or-later WITH GPL-3.0-linking-source-exception // SPDX-FileCopyrightText: 2019-2020 Normation SAS -use anyhow::Error; use std::path::PathBuf; + +use anyhow::Error; use tokio::fs::{remove_file, rename}; use tracing::debug; pub mod inventory; pub mod reporting; +pub mod shared_files; pub type ReceivedFile = PathBuf; pub type RootDirectory = PathBuf; diff --git a/relay/sources/relayd/src/processing/shared_files.rs b/relay/sources/relayd/src/processing/shared_files.rs new file mode 100644 index 00000000000..196f46f17ed --- /dev/null +++ b/relay/sources/relayd/src/processing/shared_files.rs @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: GPL-3.0-or-later WITH GPL-3.0-linking-source-exception +// SPDX-FileCopyrightText: 2019-2020 Normation SAS + +use std::{convert::TryFrom, os::unix::ffi::OsStrExt, sync::Arc}; + +use anyhow::Error; +use md5::{Digest, Md5}; +use tokio::{sync::mpsc, task::spawn_blocking}; +use tracing::{debug, error, info, span, warn, Level}; + +use crate::{ + configuration::main::SharedFilesCleanupConfig, + input::watch::*, + output::{ + database::{insert_runlog, RunlogInsertion}, + upstream::send_report, + }, + processing::{failure, success, OutputError, ReceivedFile}, + JobConfig, +}; + +pub fn start(job_config: &Arc) { + let span = span!(Level::TRACE, "shared_files"); + let _enter = span.enter(); + + let root_path = job_config.cfg.shared_files.path; + + tokio::spawn(cleanup(failed_path, job_config.cfg.shared_files.cleanup)); +} + +pub async fn cleanup(path: WatchedDirectory, cfg: SharedFilesCleanupConfig) -> Result<(), Error> { + let mut timer = interval(cfg.frequency); + + loop { + timer.tick().await; + + debug!("cleaning {:?}", path); + let sys_time = SystemTime::now(); + + let mut files = match read_dir(path.clone()).await { + Ok(f) => f, + Err(e) => { + error!("list file: {}", e); + continue; + } + }; + loop { + let entry = match files.next_entry().await { + Ok(Some(e)) => e, + // Nothing to do + Ok(None) => break, + Err(e) => { + error!("entry error: {}", e); + continue; + } + }; + let metadata = match entry.metadata().await { + Ok(m) => m, + Err(e) => { + error!("metadata error: {}", e); + continue; + } + }; + + let since = sys_time + .duration_since(metadata.modified().unwrap_or(sys_time)) + // An error indicates a file in the future, let's approximate it to now + .unwrap_or_else(|_| Duration::new(0, 0)); + + if since > cfg.retention { + let path = entry.path(); + debug!("removing old file: {:?}", path); + remove_file(path) + .await + .unwrap_or_else(|e| error!("removal error: {}", e)); + } + } + } +} diff --git a/relay/sources/relayd/tests/api_shared_files/37817c4d-fbf7-4850-a985-50021f4e8f41/files/e745a140-40bc-4b86-b6dc-084488fc906b/file-future.metadata b/relay/sources/relayd/tests/api_shared_files/37817c4d-fbf7-4850-a985-50021f4e8f41/files/e745a140-40bc-4b86-b6dc-084488fc906b/file-future.metadata new file mode 100644 index 00000000000..eded8a674c1 --- /dev/null +++ b/relay/sources/relayd/tests/api_shared_files/37817c4d-fbf7-4850-a985-50021f4e8f41/files/e745a140-40bc-4b86-b6dc-084488fc906b/file-future.metadata @@ -0,0 +1,9 @@ +file.metadatafile.metadataheader=rudder-signature-v1 +algorithm=sha512 +digest=9ae39f50bbbd3a2e529a1be61e739b814b9470a0704ef8dd7d3b78d44c2813ad062004f42dab3214a4779f8b4e8a5935cc92ff0313845377c58db605a0aaa5cffbe05e5809d4d646249cbe1fdbc6c498a7bc533415cb32de1b1f782c2f6b453b737999c4f8a15875cfe6a2808668e8c17986e100395633f702e709c2d49b7aa6eca6503e98360d58ca08ba0e91455512966a72036eeaa8c53dd016a129b2c366578c43d16c59208bdc34b39c81a8c88c073fc44bf64375644f6260d27de400ce790bd0b43251ec4640e2b9c0e2916b5e2d08395eec19f3f24a5c6bd39e2a6685f540e43c43508a98bca5cb7825022db8169080d9d3f8660ffc9b11aaa9b8088311f9bc3a0f1d822055d7ec1fedb9adabeeac7625106fa0597dfebd0d22e4703feedc2137ff705193249707e228a139015f49455fa1742e83676b5af5f84e58b71bad7846fcd731326982a4f48fd301360cc6e911554cdf193b99c7491c707b2ea373bc7ebf71d02bcdd34f4812b701977e79e7ed1b9ab6522f5e8e048cf024f24e7f0a2fcc47d6f72543acf421560da8aa9bc71c88083173122fb4905e683f4be91155d8c87f948b8b0357b10594dd61cf33dff8f3bb418a1fdc5f88ea0589906adbff1774d310829576326e4d376a8d6bc495240d030e79c76bde2daa95c051aae8b8fb2dca9327b120d494d6e9fe4dcd30cda6ba6a110cc3055a6f2a4b146d +hash_value=dda78e9b97a69aca3cff21de266246bde0d91bc4b61df72bfb0387564ac0c7bd64dd4caca39ce1ef400f32aa711ec4909789705beec93314eb65fabd5183bbfe +short_pubkey=MIICCgKCAgEAuok8JTvRssiupO0IfH4OGnWFqQg5dmI/4JsCiPEUf78iFBwFFpwuNXDJXCKaHtpjuc3DAy9l7fmZ+bQmkfde+Qo3yAd2ZsId80TBZOy6uFQyl4ASLNgY8RKIFxD6+AsutI27KexSnL3QLCgywnheRv4Ur31a6MVY1xfSQnADruBBad+5SaF3hTpEcAMg2hDQsIcyR32MPRy9MOVmvBlgI2hZsgh9QQf9wTLxGuMw/pJKOPRwwFkk/5bhFBve2sL1OI0pRsM6i7SxNXRhM6NWlmObhP+Z7C6N7TY00Z+tizgETmYJ35llyInjc1i+0bWaj5p3cbSCVdQ5zomZ3L9XbsWmjl0P/cw06qqNPuLR799K+R1XgA94nUUzo2pVigPh6sj2XMS8FOWXMXy2TNEOA+NQV5+vYwIlUizvB/HHSc3WKqNGgCifdJBmJJ8QTg5cJE6s+91O99eMMAQ0Ecj+nY5QEYkbIn4gjNpojam3jyS72o0J4nlj4ECbR/rj6L5b+kj5F3DbYqSdLC+crKUIoBZH1msCuJcQ9Zk/YHw87iVyWoZOVtJUUaw3n8vH/YCWPBQRzZp+4zlyIYJIIz+V/FJZX5YNW9XgoeRG8Q0mOmLy0FbQUS/klYlpeW3PKLSQmcSLvrgZnhKMyhEohC0zOSqJU0ui4VUWY5tv1bhbTo8CAwEAAQ== +hostname=node1.rudder.local +keydate=2020-01-24 12:17:59.014153459 +0100 +keyid=B85B4E8F +expires=2061475500 \ No newline at end of file diff --git a/relay/sources/relayd/tests/files/config/main.conf b/relay/sources/relayd/tests/files/config/main.conf index 694ab43e6f8..145e3dac963 100644 --- a/relay/sources/relayd/tests/files/config/main.conf +++ b/relay/sources/relayd/tests/files/config/main.conf @@ -59,6 +59,9 @@ use_sudo = false [shared_files] path = "tests/api_shared_files" +[shared_files.cleanup] +frequency = "43s" + [shared_folder] path = "tests/api_shared_folder" diff --git a/relay/sources/relayd/tests/server.py b/relay/sources/relayd/tests/server.py index 937453ff0c4..7112c359c8b 100755 --- a/relay/sources/relayd/tests/server.py +++ b/relay/sources/relayd/tests/server.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 from http.server import HTTPServer, BaseHTTPRequestHandler -import ssl +from ssl import SSLContext, PROTOCOL_TLS_SERVER import time from pprint import pprint import sys @@ -9,47 +9,47 @@ PORT = 4443 nodeid = sys.argv[1] + class PolicyServer(BaseHTTPRequestHandler): def do_GET(self): """Respond to a GET request.""" - if self.path == "/uuid": + if self.path == '/uuid': self.send_response(200) - self.send_header("Content-type", "text/plain") + self.send_header('Content-type', 'text/plain') self.end_headers() self.wfile.write(str.encode(nodeid)) - elif self.path == "/stop": + elif self.path == '/stop': self.send_response(200) - self.send_header("Content-type", "text/plain") + self.send_header('Content-type', 'text/plain') self.end_headers() - self.wfile.write(b"test server stopping\n") + self.wfile.write(b'test server stopping\n') exit(0) else: self.send_error(404) def do_POST(self): """Respond to a POST request.""" - if self.path == "/rudder/relay-api/remote-run/nodes": + if self.path == '/rudder/relay-api/remote-run/nodes': time.sleep(0.2) self.send_response(200) - self.send_header("Content-type", "text/plain") + self.send_header('Content-type', 'text/plain') self.end_headers() - self.wfile.write(b"REMOTE\n") - f = open("target/tmp/api_test_remote.txt", "w") + self.wfile.write(b'REMOTE\n') + f = open('target/tmp/api_test_remote.txt', 'w') # TODO also write received parameters - f.write("OK") + f.write('OK') f.close() else: self.send_error(404) + server_address = ('', PORT) httpd = HTTPServer(server_address, PolicyServer) -httpd.socket = ssl.wrap_socket( - httpd.socket, - server_side=True, - certfile='tests/files/keys/' + - nodeid + - '.cert', - keyfile='tests/files/keys/' + - nodeid + - '.nopass.priv') +context = SSLContext(PROTOCOL_TLS_SERVER) +context.check_hostname = False +context.load_cert_chain( + certfile='tests/files/keys/' + nodeid + '.cert', + keyfile='tests/files/keys/' + nodeid + '.nopass.priv', +) +httpd.socket = context.wrap_socket(httpd.socket, server_side=True) httpd.serve_forever() diff --git a/relay/sources/relayd/tools/config/main.conf b/relay/sources/relayd/tools/config/main.conf index 837f097d7d2..678453212c9 100644 --- a/relay/sources/relayd/tools/config/main.conf +++ b/relay/sources/relayd/tools/config/main.conf @@ -135,6 +135,10 @@ password = "PASSWORD" # Path of files shared between individual nodes #path = "/var/rudder/shared-files/" +[shared_files.cleanup] +# Job frequency +#frequency = "10min" + [shared_folder] # Path of files shared to all the nodes #path = "/var/rudder/configuration-repository/shared-files" diff --git a/relay/sources/rudder-relay.cron b/relay/sources/rudder-relay.cron deleted file mode 100644 index eaf3620d73a..00000000000 --- a/relay/sources/rudder-relay.cron +++ /dev/null @@ -1,5 +0,0 @@ -# Cron file for Rudder relay -# - -*/5 * * * * root /opt/rudder/bin/relay-cleanup >/dev/null -