From f98f5051515dcd48f31670659cab331687658e48 Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 23 Oct 2020 23:40:59 -0400 Subject: [PATCH 1/4] Adds unzip module Allows downloading and extracting zips from urls --- Cargo.lock | 60 +++++++++++++++++++++++++++++--------------- Cargo.toml | 2 ++ build.rs | 8 ++++++ src/error.rs | 6 +++++ src/http.rs | 2 +- src/lib.rs | 2 ++ src/unzip.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 130 insertions(+), 21 deletions(-) create mode 100644 src/unzip.rs diff --git a/Cargo.lock b/Cargo.lock index 38bb2eec..6c7a163a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,11 +1,5 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -[[package]] -name = "adler" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" - [[package]] name = "adler32" version = "1.2.0" @@ -132,6 +126,27 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" +[[package]] +name = "bzip2" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42b7c3cbf0fa9c1b82308d57191728ca0256cb821220f4e2fd410a72ade26e3b" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.9+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad3b39a260062fca31f7b0b12f207e8f2590a67d32ec7d59c20484b07ea7285e" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "cc" version = "1.0.61" @@ -384,15 +399,15 @@ checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" [[package]] name = "flate2" -version = "1.0.18" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da80be589a72651dcda34d8b35bcdc9b7254ad06325611074d9cc0fbb19f60ee" +checksum = "2cfff41391129e0a856d6d822600b8d71179d46879e310417eb9c762eb178b42" dependencies = [ "cfg-if", "crc32fast", "libc", "libz-sys", - "miniz_oxide 0.4.3", + "miniz_oxide", ] [[package]] @@ -975,16 +990,6 @@ dependencies = [ "adler32", ] -[[package]] -name = "miniz_oxide" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" -dependencies = [ - "adler", - "autocfg", -] - [[package]] name = "mio" version = "0.6.22" @@ -1313,7 +1318,7 @@ dependencies = [ "bitflags", "crc32fast", "deflate 0.8.6", - "miniz_oxide 0.3.7", + "miniz_oxide", ] [[package]] @@ -1553,6 +1558,7 @@ dependencies = [ "sha2 0.9.1", "thiserror", "url", + "zip", ] [[package]] @@ -2287,3 +2293,17 @@ dependencies = [ "winapi 0.2.8", "winapi-build", ] + +[[package]] +name = "zip" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "543adf038106b64cfca4711c82c917d785e3540e04f7996554488f988ec43124" +dependencies = [ + "byteorder", + "bzip2", + "crc32fast", + "flate2", + "thiserror", + "time 0.1.44", +] diff --git a/Cargo.toml b/Cargo.toml index 89fd48dc..e952f5bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ lazy_static = { version = "1.4", optional = true } once_cell = { version = "1.4", optional = true } mysql = { version = "20.0", optional = true } dashmap = { version = "3.11", optional = true } +zip = { version = "0.5.8", optional = true } [features] @@ -48,3 +49,4 @@ url = ["url-dep", "percent-encoding"] git = ["git2", "chrono"] http = ["reqwest", "serde", "serde_json", "once_cell"] sql = ["mysql", "serde", "serde_json", "once_cell", "dashmap"] +unzip = ["zip"] diff --git a/build.rs b/build.rs index f4a2bae2..8ab68150 100644 --- a/build.rs +++ b/build.rs @@ -188,6 +188,14 @@ fn main() { #define rustg_sql_connected(handle) call(RUST_G, "sql_connected")(handle) #define rustg_sql_disconnect_pool(handle) call(RUST_G, "sql_disconnect_pool")(handle) #define rustg_sql_check_query(job_id) call(RUST_G, "sql_check_query")("[job_id]") +"#).unwrap(); + } + + // module: zip + if enabled!("UNZIP") { + write!(f, r#" +#define rustg_unzip_download_async(url, unzip_directory) call(RUST_G, "unzip_download_async")(url, unzip_directory) +#define rustg_unzip_check(job_id) call(RUST_G, "unzip_check")("[job_id]") "#).unwrap(); } } diff --git a/src/error.rs b/src/error.rs index 2ccfe27f..2178be5e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -9,6 +9,9 @@ use thiserror::Error; #[cfg(feature = "png")] use png::{DecodingError, EncodingError}; +#[cfg(feature = "unzip")] +use zip::result::ZipError; + pub type Result = result::Result; #[derive(Error, Debug)] @@ -42,6 +45,9 @@ pub enum Error { #[cfg(feature = "http")] #[error(transparent)] SerializationError(#[from] serde_json::Error), + #[cfg(feature = "unzip")] + #[error(transparent)] + UnzipError(#[from] ZipError) } impl From for Error { diff --git a/src/http.rs b/src/http.rs index 452f3d18..28c8345e 100644 --- a/src/http.rs +++ b/src/http.rs @@ -76,7 +76,7 @@ fn setup_http_client() -> reqwest::blocking::Client { Client::builder().default_headers(headers).build().unwrap() } -static HTTP_CLIENT: Lazy = Lazy::new(setup_http_client); +pub static HTTP_CLIENT: Lazy = Lazy::new(setup_http_client); // ---------------------------------------------------------------------------- // Request construction and execution diff --git a/src/lib.rs b/src/lib.rs index eef27a0b..923463ac 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,8 @@ pub mod noise_gen; pub mod sql; #[cfg(feature = "url")] pub mod url; +#[cfg(feature = "unzip")] +pub mod unzip; #[cfg(not(target_pointer_width = "32"))] compile_error!("rust-g must be compiled for a 32-bit target"); diff --git a/src/unzip.rs b/src/unzip.rs new file mode 100644 index 00000000..6b9f7af7 --- /dev/null +++ b/src/unzip.rs @@ -0,0 +1,71 @@ +use crate::{error::Result, jobs, http::HTTP_CLIENT}; +use reqwest::blocking::RequestBuilder; +use std::fs; +use std::io::Write; +use std::path::Path; +use zip::ZipArchive; + +struct UnzipPrep { + req: RequestBuilder, + unzip_directory: String, +} + +fn construct_unzip( + url: &str, + unzip_directory: &str +) -> UnzipPrep { + + let req = HTTP_CLIENT.get(url); + let dir_copy = unzip_directory.to_string(); + + UnzipPrep { + req, + unzip_directory: dir_copy, + } +} + +byond_fn! { unzip_download_async(url, unzip_directory) { + let unzip = construct_unzip(&url, &unzip_directory); + Some(jobs::start(move || + match do_unzip_download(unzip) { + Ok(r) => r, + Err(e) => e.to_string() + } + )) +} } + +fn do_unzip_download(prep: UnzipPrep) -> Result { + let unzip_path = Path::new(&prep.unzip_directory); + let response = prep.req.send()?; + + let content = response.bytes()?; + + let reader = std::io::Cursor::new(content); + let mut archive = ZipArchive::new(reader)?; + + for i in 0..archive.len() + { + let mut entry = archive.by_index(i).unwrap(); + + let file_path = unzip_path.join(entry.name()); + + if let Some(parent) = file_path.parent() { + fs::create_dir_all(parent)? + } + + let file = fs::OpenOptions::new() + .write(true) + .create(true) + .open(&file_path)?; + + let mut writer = std::io::BufWriter::new(file); + std::io::copy(&mut entry, &mut writer)?; + writer.flush()?; + } + + Ok("true".to_string()) +} + +byond_fn! { unzip_check(id) { + Some(jobs::check(id)) +} } From 62ae3b06d78a398587b3014eafb87f316609a1cc Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Fri, 23 Oct 2020 23:50:57 -0400 Subject: [PATCH 2/4] unwrap... OR ELSE --- src/unzip.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/unzip.rs b/src/unzip.rs index 6b9f7af7..e667158c 100644 --- a/src/unzip.rs +++ b/src/unzip.rs @@ -27,10 +27,7 @@ fn construct_unzip( byond_fn! { unzip_download_async(url, unzip_directory) { let unzip = construct_unzip(&url, &unzip_directory); Some(jobs::start(move || - match do_unzip_download(unzip) { - Ok(r) => r, - Err(e) => e.to_string() - } + do_unzip_download(unzip).unwrap_or_else(|e| e.to_string()) )) } } From 5106e019bebe3fc8aea25b8b3dd9f7575cf0599e Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Sat, 24 Oct 2020 21:15:06 -0400 Subject: [PATCH 3/4] Convert to new build script style --- build.rs | 8 -------- dmsrc/unzip.dm | 2 ++ 2 files changed, 2 insertions(+), 8 deletions(-) create mode 100644 dmsrc/unzip.dm diff --git a/build.rs b/build.rs index f8ce03e5..99d60ebf 100644 --- a/build.rs +++ b/build.rs @@ -34,12 +34,4 @@ fn main() { } } } - - // module: zip - if enabled!("UNZIP") { - write!(f, r#" -#define rustg_unzip_download_async(url, unzip_directory) call(RUST_G, "unzip_download_async")(url, unzip_directory) -#define rustg_unzip_check(job_id) call(RUST_G, "unzip_check")("[job_id]") -"#).unwrap(); - } } diff --git a/dmsrc/unzip.dm b/dmsrc/unzip.dm new file mode 100644 index 00000000..be56e5f3 --- /dev/null +++ b/dmsrc/unzip.dm @@ -0,0 +1,2 @@ +#define rustg_unzip_download_async(url, unzip_directory) call(RUST_G, "unzip_download_async")(url, unzip_directory) +#define rustg_unzip_check(job_id) call(RUST_G, "unzip_check")("[job_id]") \ No newline at end of file From bad302f64e89d48a2ad9298703d47831c26aacaa Mon Sep 17 00:00:00 2001 From: Jordan Brown Date: Mon, 26 Oct 2020 14:10:00 -0400 Subject: [PATCH 4/4] Update src/unzip.rs Co-authored-by: Rohesie --- src/unzip.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/unzip.rs b/src/unzip.rs index e667158c..48a17a76 100644 --- a/src/unzip.rs +++ b/src/unzip.rs @@ -42,7 +42,7 @@ fn do_unzip_download(prep: UnzipPrep) -> Result { for i in 0..archive.len() { - let mut entry = archive.by_index(i).unwrap(); + let mut entry = archive.by_index(i)?; let file_path = unzip_path.join(entry.name());