From 1d936f15a3816fc6d0b0c276b17c53d62885634b Mon Sep 17 00:00:00 2001 From: Armand Sylvain Date: Fri, 19 Jul 2019 17:09:43 +0200 Subject: [PATCH] Fixes #15253: Sending & Receiving files with the shared-files API --- relay/sources/relayd/Cargo.lock | 17 +++ relay/sources/relayd/Cargo.toml | 2 + relay/sources/relayd/src/api.rs | 39 +++-- relay/sources/relayd/src/error.rs | 2 + relay/sources/relayd/src/shared_files.rs | 173 ++++++++++++++++------- 5 files changed, 173 insertions(+), 60 deletions(-) diff --git a/relay/sources/relayd/Cargo.lock b/relay/sources/relayd/Cargo.lock index 9ff1c9a343c..6b104696bd5 100644 --- a/relay/sources/relayd/Cargo.lock +++ b/relay/sources/relayd/Cargo.lock @@ -116,6 +116,19 @@ name = "byte-tools" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bytebuffer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "byteorder" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "byteorder" version = "1.3.2" @@ -1421,6 +1434,8 @@ dependencies = [ name = "relayd" version = "0.0.0-dev" dependencies = [ + "bytebuffer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "criterion 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "diesel 1.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2357,6 +2372,8 @@ dependencies = [ "checksum block-padding 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "6d4dc3af3ee2e12f3e5d224e5e1e3d73668abbeb69e566d361f7d5563a4fdf09" "checksum bstr 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "692188621cda27df291548782a09931465799ee047fa07b44a764002cae57f4b" "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" +"checksum bytebuffer 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e822a21389d388828152aeae8bb43049196b09076e2a138f53351d8cf6576cf3" +"checksum byteorder 0.3.13 (registry+https://github.com/rust-lang/crates.io-index)" = "29b2aa490a8f546381308d68fc79e6bd753cd3ad839f7a7172897f1feedfa175" "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum c2-chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7d64d04786e0f528460fc884753cf8dddcc466be308f6026f8e355c41a0e4101" diff --git a/relay/sources/relayd/Cargo.toml b/relay/sources/relayd/Cargo.toml index 04d55c37d56..23be49b1474 100644 --- a/relay/sources/relayd/Cargo.toml +++ b/relay/sources/relayd/Cargo.toml @@ -51,6 +51,8 @@ itertools = "0.8" md-5 = "0.8" hyper = "0.12" sha2 = "0.8" +bytebuffer = "0.2.1" +bytes = "0.4.12" [dev-dependencies] criterion = "0.2" diff --git a/relay/sources/relayd/src/api.rs b/relay/sources/relayd/src/api.rs index 111988bd5dc..aca0b7d0b2e 100644 --- a/relay/sources/relayd/src/api.rs +++ b/relay/sources/relayd/src/api.rs @@ -31,9 +31,13 @@ use crate::{ error::Error, remote_run::{RemoteRun, RemoteRunTarget}, - shared_files::{metadata_hash_checker, metadata_writer, parse_hash_from_raw, Metadata}, + shared_files::{ + file_writer, metadata_hash_checker, metadata_parser, metadata_writer, + parse_parameter_from_raw, parse_ttl, + }, {stats::Stats, status::Status, JobConfig}, }; + use futures::Future; use std::{ collections::HashMap, @@ -41,7 +45,7 @@ use std::{ sync::{Arc, RwLock}, }; use tracing::info; -use warp::{body, filters, filters::method::v2::*, path, reject::custom, reply, Filter}; +use warp::{body, filters, filters::method::v2::*, path, reject::custom, reply, Buf, Filter}; pub fn api( listen: SocketAddr, @@ -111,13 +115,25 @@ pub fn api( }, ); - let shared_files_put = put().and(path::peek()).and(body::form()).map( - move |peek: filters::path::Peek, simple_map: HashMap| { - let metadata = Metadata::new(simple_map); - metadata_writer(format!("{}", metadata.unwrap()), peek.as_str()); - reply() - }, - ); + let shared_files_put = put() + .and(filters::query::raw()) + .and(path::peek()) + .and(warp::body::concat()) + .map( + move |ttl: String, peek: filters::path::Peek, mut buf: warp::body::FullBody| { + metadata_writer( + format!( + "{}\nexpires={}", + metadata_parser(buf.by_ref()).join("\n"), + parse_ttl(parse_parameter_from_raw(ttl)).unwrap() + ), + peek.as_str(), + ); + + file_writer(buf.by_ref(), peek.as_str()); + reply() + }, + ); let shared_files_head = head() .and(path::peek()) @@ -125,7 +141,10 @@ pub fn api( .map(|peek: filters::path::Peek, raw: String| { reply::with_status( "".to_string(), - metadata_hash_checker(format!("./{}", peek.as_str()), parse_hash_from_raw(raw)), + metadata_hash_checker( + format!("./{}", peek.as_str()), + parse_parameter_from_raw(raw), + ), ) }); diff --git a/relay/sources/relayd/src/error.rs b/relay/sources/relayd/src/error.rs index c7419bad50f..aa81c3b0f87 100644 --- a/relay/sources/relayd/src/error.rs +++ b/relay/sources/relayd/src/error.rs @@ -75,6 +75,7 @@ pub enum Error { MissingTargetNodes, InvalidHashType, InvalidLogFilter(tracing_fmt::filter::env::ParseError), + InvalidHeader, } impl Display for Error { @@ -116,6 +117,7 @@ impl Display for Error { "Invalid hash type provided, available hash types : sha256, sha512" ), InvalidLogFilter(ref err) => write!(f, "Log filter is invalid: {}", err), + InvalidHeader => write!(f, "Invalid header"), } } } diff --git a/relay/sources/relayd/src/shared_files.rs b/relay/sources/relayd/src/shared_files.rs index 3b25608fdfc..7bbc68a84c6 100644 --- a/relay/sources/relayd/src/shared_files.rs +++ b/relay/sources/relayd/src/shared_files.rs @@ -39,12 +39,15 @@ use openssl::pkey::Public; use openssl::rsa::Rsa; use openssl::sign::Verifier; use regex::Regex; +use reqwest; use sha2::{Digest, Sha256, Sha512}; -use std::collections::HashMap; use std::fmt; use std::fs; +use std::io::BufRead; +use std::io::Read; use std::path::Path; use std::str::FromStr; +use warp::Buf; pub enum HashType { Sha256, @@ -89,49 +92,15 @@ impl HashType { #[derive(Debug)] pub struct Metadata { - header: String, - algorithm: String, - digest: String, - hash_value: String, - short_pubkey: String, - hostname: String, - keydate: String, - keyid: String, - expires: String, -} - -impl FromStr for Metadata { - type Err = Error; - - fn from_str(s: &str) -> Result { - Ok(Metadata { - header: parse_value("header", s).unwrap(), - algorithm: parse_value("algorithm", s).unwrap(), - digest: parse_value("digest", s).unwrap(), - hash_value: parse_value("hash_value", s).unwrap(), - short_pubkey: parse_value("short_pubkey", s).unwrap(), - hostname: parse_value("hostname", s).unwrap(), - keydate: parse_value("keydate", s).unwrap(), - keyid: parse_value("keyid", s).unwrap(), - expires: parse_value("expires", s).unwrap(), - }) - } -} - -impl Metadata { - pub fn new(hashmap: HashMap) -> Result { - Ok(Metadata { - header: hashmap.get("header").unwrap().to_string(), - algorithm: hashmap.get("algorithm").unwrap().to_string(), - digest: hashmap.get("digest").unwrap().to_string(), - hash_value: hashmap.get("hash_value").unwrap().to_string(), - short_pubkey: hashmap.get("short_pubkey").unwrap().to_string(), - hostname: hashmap.get("hostname").unwrap().to_string(), - keydate: hashmap.get("keydate").unwrap().to_string(), - keyid: hashmap.get("keyid").unwrap().to_string(), - expires: hashmap.get("expires").unwrap().to_string(), - }) - } + pub header: String, + pub algorithm: String, + pub digest: String, + pub hash_value: String, + pub short_pubkey: String, + pub hostname: String, + pub keydate: String, + pub keyid: String, + pub expires: String, } impl fmt::Display for Metadata { @@ -154,6 +123,27 @@ impl fmt::Display for Metadata { } } +impl FromStr for Metadata { + type Err = Error; + + fn from_str(s: &str) -> Result { + if parse_value("header", s).unwrap() != "rudder-signature-v1" { + return Err(Error::InvalidHeader); + } + Ok(Metadata { + header: "rudder-signature-v1".to_string(), + algorithm: parse_value("algorithm", s).unwrap(), + digest: parse_value("digest", s).unwrap(), + hash_value: parse_value("hash_value", s).unwrap(), + short_pubkey: parse_value("short_pubkey", s).unwrap(), + hostname: parse_value("hostname", s).unwrap(), + keydate: parse_value("keydate", s).unwrap(), + keyid: parse_value("keyid", s).unwrap(), + expires: parse_value("expires", s).unwrap(), + }) + } +} + pub fn parse_value(key: &str, file: &str) -> Result { let regex_key = Regex::new(&format!(r"{}=(?P[^\n]+)\n", key)).unwrap(); @@ -198,7 +188,7 @@ pub fn validate_signature( } pub fn parse_ttl(ttl: String) -> Result { - let regex_numbers = Regex::new(r"^(?:(?P\d+)(?:d|days))?\s*(?:(?P\d+)(?:h|hours))?\s*(?:(?P\d+)(?:m|minutes))?\s*(?:(?P\d+)(?:s|seconds))?").unwrap(); + let regex_numbers = Regex::new(r"^(?:(?P\d+)(?:d|days|day))?\s*(?:(?P\d+)(?:h|hours|hour))?\s*(?:(?P\d+)(?:m|minutes|minute))?\s*(?:(?P\d+)(?:s|seconds|second))?").unwrap(); fn parse_time<'t>(cap: ®ex::Captures<'t>, n: &str) -> Result { Ok(match cap.name(n) { @@ -225,12 +215,38 @@ pub fn parse_ttl(ttl: String) -> Result { } } +pub fn metadata_parser(buf: &mut warp::body::FullBody) -> Vec { + let mut metadata = Vec::new(); + + let reader = buf.reader(); + + for line in reader.lines() { + let mytmpstr = line.unwrap(); + if mytmpstr != "" { + metadata.push(mytmpstr); + } else { + break; + } + } + metadata +} + +pub fn file_writer(buf: &mut warp::body::FullBody, path: &str) { + let mut myvec: Vec = vec![]; + + buf.by_ref().reader().consume(1); // skip the line feed + buf.reader().read_to_end(&mut myvec).unwrap(); + + fs::write(format!("shared-files/{}", path), myvec).expect("Unable to write file"); +} + pub fn metadata_writer(metadata_string: String, peek: &str) { let myvec: Vec = peek.split('/').map(|s| s.to_string()).collect(); let (target_uuid, source_uuid, _file_id) = (&myvec[0], &myvec[1], &myvec[2]); - let _ = fs::create_dir_all(format!("./{}/{}/", target_uuid, source_uuid)); // on cree les folders s'ils existent pas - //fs::create_dir_all(format!("/var/rudder/configuration-repository/shared-files/{}/{}/", target_uuid, source_uuid)); // real path - fs::write(format!("./{}", peek), metadata_string).expect("Unable to write file"); + let _ = fs::create_dir_all(format!("shared-files/{}/{}/", target_uuid, source_uuid)); // on cree les folders s'ils existent pas + //fs::create_dir_all(format!("/var/rudder/configuration-repository/shared-files/{}/{}/", target_uuid, source_uuid)); // real path + fs::write(format!("shared-files/{}.metadata", peek), metadata_string) + .expect("Unable to write file"); } pub fn metadata_hash_checker(filename: String, hash: String) -> hyper::StatusCode { @@ -255,13 +271,48 @@ pub fn metadata_hash_checker(filename: String, hash: String) -> hyper::StatusCod StatusCode::from_u16(404).unwrap() } -pub fn parse_hash_from_raw(raw: String) -> String { +pub fn parse_parameter_from_raw(raw: String) -> String { raw.split('=') .map(|s| s.to_string()) - .filter(|s| s != "hash") + .filter(|s| s != "hash" || s != "ttl") .collect::() } +#[cfg(test)] +mod tests { + use super::*; + use openssl::sign::Signer; + + #[test] + pub fn it_writes_the_metadata() { + let metadata = Metadata { + header: "rudder-signature-v1".to_string(), + algorithm: "sha256".to_string(), + digest: "8ca9efc5752e133e2e80e2661c176fa50f".to_string(), + hash_value: "a75fda39a7af33eb93ab1c74874dcf66d5761ad30977368cf0c4788cf5bfd34f" + .to_string(), + short_pubkey: "shortpubkey".to_string(), + hostname: "ubuntu-18-04-64".to_string(), + keydate: "2018-10-3118:21:43.653257143".to_string(), + keyid: "B29D02BB".to_string(), + expires: "1d 1h".to_string(), + }; + + assert_eq!(format!("{}", metadata), format!("header=rudder-signature-v1\nalgorithm=sha256\ndigest=8ca9efc5752e133e2e80e2661c176fa50f\nhash_value=a75fda39a7af33eb93ab1c74874dcf66d5761ad30977368cf0c4788cf5bfd34f\nshort_pubkey=shortpubkey\nhostname=ubuntu-18-04-64\nkeydate=2018-10-3118:21:43.653257143\nkeyid=B29D02BB\nexpires={}\n", parse_ttl("1d 1h".to_string()).unwrap())); + } + +pub fn send_file(file_id: String, source_uuid: String, target_uuid: String) { + let file_test = fs::File::open("target/tmp/test_send_file.txt").unwrap(); + let client = reqwest::Client::new(); + let _res = client + .put(&format!( + "https://relay/rudder/relay-api/shared-files/{}/{}/{}", + target_uuid, source_uuid, file_id + )) + .body(file_test) + .send(); +} + #[cfg(test)] mod tests { use super::*; @@ -319,4 +370,26 @@ z5VEb9yx2KikbWyChM1Akp82AV5BzqE80QIBIw==".to_string()).unwrap(), assert!(validate_signature(data, keypub, HashType::Sha512, &signature).unwrap()); } + + #[test] + pub fn it_validates_signatures() { + // Generate a keypair + let k0 = Rsa::generate(2048).unwrap(); + let k0pkey = k0.public_key_to_pem().unwrap(); + let k1 = Rsa::public_key_from_pem(&k0pkey).unwrap(); + + let keypriv = PKey::from_rsa(k0).unwrap(); + let keypub = PKey::from_rsa(k1).unwrap(); + + let data = b"hello, world!"; + + // Sign the data + let mut signer = Signer::new(HashType::Sha512.to_openssl_hash(), &keypriv).unwrap(); + signer.update(data).unwrap(); + + let signature = signer.sign_to_vec().unwrap(); + + assert!(validate_signature(data, keypub, HashType::Sha512, &signature).unwrap()); + } + }