diff --git a/src/config.rs b/src/config.rs index 3353e71f..5b9ca5d7 100644 --- a/src/config.rs +++ b/src/config.rs @@ -2,6 +2,7 @@ use std::time::Duration; use moka::future::Cache; +use crate::storage::{PrivateStorage, StaticStorage}; use crate::{ endpoints::mods::IndexQueryParams, types::{ @@ -17,6 +18,8 @@ pub struct AppData { front_url: String, github: GitHubClientData, webhook_url: String, + static_storage: StaticStorage, + private_storage: PrivateStorage, disable_downloads: bool, max_download_mb: u32, port: u16, @@ -34,7 +37,8 @@ pub struct GitHubClientData { pub async fn build_config() -> anyhow::Result { let env_url = dotenvy::var("DATABASE_URL")?; - let pg_connections = dotenvy::var("DATABASE_CONNECTIONS").map_or(10, |x: String| x.parse::().unwrap_or(10)); + let pg_connections = + dotenvy::var("DATABASE_CONNECTIONS").map_or(10, |x: String| x.parse::().unwrap_or(10)); let pool = sqlx::postgres::PgPoolOptions::default() .max_connections(pg_connections) @@ -68,6 +72,8 @@ pub async fn build_config() -> anyhow::Result { client_secret: github_secret, }, webhook_url, + static_storage: StaticStorage::new(), + private_storage: PrivateStorage::new(), disable_downloads, max_download_mb, port, @@ -123,6 +129,14 @@ impl AppData { self.debug } + pub fn static_storage(&self) -> &StaticStorage { + &self.static_storage + } + + pub fn private_storage(&self) -> &PrivateStorage { + &self.private_storage + } + pub fn mods_cache(&self) -> &Cache>> { &self.mods_cache } diff --git a/src/main.rs b/src/main.rs index 32ac1f7b..081182d3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ mod mod_zip; mod openapi; mod types; mod webhook; +mod storage; #[tokio::main] async fn main() -> anyhow::Result<()> { diff --git a/src/storage.rs b/src/storage.rs new file mode 100644 index 00000000..d8ab9770 --- /dev/null +++ b/src/storage.rs @@ -0,0 +1,106 @@ +use std::path::{Path, PathBuf}; + +#[derive(Clone, Debug)] +pub struct StaticStorage { + base_path: PathBuf, +} + +impl StaticStorage { + pub fn new() -> Self { + Self { + base_path: PathBuf::from("storage/static"), + } + } +} + +impl StorageDisk for StaticStorage { + fn base_path(&self) -> &Path { + &self.base_path + } +} + +#[derive(Clone, Debug)] +pub struct PrivateStorage { + base_path: PathBuf, +} + +impl PrivateStorage { + pub fn new() -> Self { + Self { + base_path: PathBuf::from("storage/private"), + } + } +} + +impl StorageDisk for PrivateStorage { + fn base_path(&self) -> &Path { + &self.base_path + } +} + +trait StorageDisk { + async fn init(&self) -> std::io::Result<()> { + tokio::fs::create_dir_all(self.base_path()).await?; + Ok(()) + } + fn base_path(&self) -> &Path; + fn path(&self, relative_path: &str) -> PathBuf { + self.base_path().join(relative_path) + } + async fn store(&self, relative_path: &str, data: &[u8]) -> std::io::Result<()> { + let path = self.path(relative_path); + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + + tokio::fs::write(path, data).await + } + /// Store data at a path calculated from the hash of the data. Uses content-addressable storage with 2 levels + async fn store_hashed(&self, relative_path: &str, data: &[u8]) -> std::io::Result<()> { + self.store_hashed_with_extension(relative_path, data, None) + .await + } + /// Store data at a path calculated from the hash of the data. Uses content-addressable storage with 2 levels. + /// Extension should not include the dot, and will be added to the end of the filename if provided. + async fn store_hashed_with_extension( + &self, + relative_path: &str, + data: &[u8], + extension: Option<&str>, + ) -> std::io::Result<()> { + let hash = sha256::digest(data); + + let hashed_path = format!( + "{}/{}/{}{}", + relative_path, + &hash[0..2], + hash, + extension.map_or("".to_string(), |ext| format!( + ".{}", + ext.trim_start_matches('.') + )) + ); + self.store(&hashed_path, data).await + } + async fn read(&self, relative_path: &str) -> std::io::Result> { + match tokio::fs::read(self.path(relative_path)).await { + Ok(data) => Ok(data), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(vec![]), + Err(e) => Err(e), + } + } + async fn read_stream(&self, relative_path: &str) -> std::io::Result { + tokio::fs::File::open(self.path(relative_path)).await + } + async fn exists(&self, relative_path: &str) -> std::io::Result { + let path = self.path(relative_path); + Ok(tokio::fs::metadata(path).await.is_ok()) + } + async fn delete(&self, relative_path: &str) -> std::io::Result<()> { + match tokio::fs::remove_file(self.path(relative_path)).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } + } +} diff --git a/storage/.gitignore b/storage/.gitignore new file mode 100644 index 00000000..379ee272 --- /dev/null +++ b/storage/.gitignore @@ -0,0 +1,4 @@ +! +!.gitignore +!public +!private \ No newline at end of file diff --git a/storage/private/.gitignore b/storage/private/.gitignore new file mode 100644 index 00000000..c96a04f0 --- /dev/null +++ b/storage/private/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/storage/public/.gitignore b/storage/public/.gitignore new file mode 100644 index 00000000..c96a04f0 --- /dev/null +++ b/storage/public/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file