From 67b72234d3f6005adea2595e3dcd63e6f6dceabf Mon Sep 17 00:00:00 2001 From: Valerian G Date: Sat, 9 Jul 2022 23:34:19 +0200 Subject: [PATCH] removed old scanner code --- Cargo.lock | 245 ++++++++ dim/src/external/mock.rs | 2 +- dim/src/lib.rs | 2 - .../scanner_daemon.rs => scanner/daemon.rs} | 0 dim/src/scanners/base.rs | 472 -------------- dim/src/scanners/mod.rs | 251 -------- dim/src/scanners/movie.rs | 218 ------- dim/src/scanners/tmdb.rs | 576 ------------------ dim/src/scanners/tv_show.rs | 366 ----------- 9 files changed, 246 insertions(+), 1886 deletions(-) rename dim/src/{scanners/scanner_daemon.rs => scanner/daemon.rs} (100%) delete mode 100644 dim/src/scanners/base.rs delete mode 100644 dim/src/scanners/mod.rs delete mode 100644 dim/src/scanners/movie.rs delete mode 100644 dim/src/scanners/tmdb.rs delete mode 100644 dim/src/scanners/tv_show.rs diff --git a/Cargo.lock b/Cargo.lock index 4592f5419..debae25cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler32" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" + [[package]] name = "aead" version = "0.4.3" @@ -168,6 +174,12 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +[[package]] +name = "bit_field" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4" + [[package]] name = "bitflags" version = "1.3.2" @@ -220,6 +232,12 @@ version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +[[package]] +name = "bytemuck" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5715e491b5a1598fc2bef5a606847b5dc1d48ea625bd3c02c00de8285591da" + [[package]] name = "byteorder" version = "1.4.3" @@ -348,6 +366,12 @@ dependencies = [ "vec_map", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -544,6 +568,18 @@ dependencies = [ "libc", ] +[[package]] +name = "dashmap" +version = "5.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3495912c9c1ccf2e18976439f4443f3fee0fd61f424ff99fde6a66b15ecb448f" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.12.1", + "lock_api", + "parking_lot_core 0.9.3", +] + [[package]] name = "database" version = "0.1.0" @@ -568,6 +604,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "deflate" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c86f7e25f518f4b81808a2cf1c50996a61f5c2eb394b2393bd87f2a4780a432f" +dependencies = [ + "adler32", +] + [[package]] name = "dia-i18n" version = "0.9.0" @@ -604,20 +649,25 @@ dependencies = [ "cfg-if 1.0.0", "chrono", "criterion", + "dashmap", "database", "dia-i18n", "displaydoc", + "dominant_color", "events", "fs_extra", "futures", "fuzzy-matcher", + "governor", "http", + "image", "itertools", "lazy_static", "nightfall", "nix 0.20.0", "notify", "once_cell", + "parking_lot 0.12.1", "percent-encoding", "priority-queue", "rand 0.7.3", @@ -660,6 +710,12 @@ dependencies = [ "syn", ] +[[package]] +name = "dominant_color" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2494b5d589b485e8b75ef5e81b4dc271fb0d5814132c308d5d11f0de9b300d96" + [[package]] name = "dotenv" version = "0.15.0" @@ -709,6 +765,22 @@ dependencies = [ "serde_json", ] +[[package]] +name = "exr" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14cc0e06fb5f67e5d6beadf3a382fec9baca1aa751c6d5368fdeee7e5932c215" +dependencies = [ + "bit_field", + "deflate", + "flume", + "half", + "inflate", + "lebe", + "smallvec", + "threadpool", +] + [[package]] name = "fastrand" version = "1.7.0" @@ -748,6 +820,7 @@ checksum = "1ceeb589a3157cac0ab8cc585feb749bd2cea5cb55a6ee802ad72d9fd38303da" dependencies = [ "futures-core", "futures-sink", + "nanorand", "pin-project", "spin 0.9.3", ] @@ -952,8 +1025,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" dependencies = [ "cfg-if 1.0.0", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -966,6 +1041,33 @@ dependencies = [ "polyval", ] +[[package]] +name = "gif" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3edd93c6756b4dfaf2709eafcc345ba2636565295c198a9cfbf75fa5e3e00b06" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "governor" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19775995ee20209163239355bc3ad2f33f83da35d9ef72dea26e5af753552c87" +dependencies = [ + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot 0.12.1", + "quanta", + "rand 0.8.5", + "smallvec", +] + [[package]] name = "h2" version = "0.3.13" @@ -1155,6 +1257,25 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "image" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e30ca2ecf7666107ff827a8e481de6a132a9b687ed3bb20bb1c144a36c00964" +dependencies = [ + "bytemuck", + "byteorder", + "color_quant", + "exr", + "gif", + "jpeg-decoder", + "num-rational", + "num-traits", + "png", + "scoped_threadpool", + "tiff", +] + [[package]] name = "indexmap" version = "1.9.1" @@ -1165,6 +1286,15 @@ dependencies = [ "hashbrown 0.12.1", ] +[[package]] +name = "inflate" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff" +dependencies = [ + "adler32", +] + [[package]] name = "inotify" version = "0.7.1" @@ -1239,6 +1369,15 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9478aa10f73e7528198d75109c8be5cd7d15fb530238040148d5f9a22d4c5b3b" +dependencies = [ + "rayon", +] + [[package]] name = "js-sys" version = "0.3.58" @@ -1270,6 +1409,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libc" version = "0.2.126" @@ -1474,6 +1619,15 @@ dependencies = [ "twoway", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.7", +] + [[package]] name = "net2" version = "0.2.37" @@ -1537,6 +1691,12 @@ dependencies = [ "memoffset", ] +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + [[package]] name = "nom" version = "7.1.1" @@ -1547,6 +1707,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "notify" version = "4.0.17" @@ -1768,6 +1934,18 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc38c0ad57efb786dd57b9864e5b18bae478c00c824dc55a38bbc9da95dde3ba" +dependencies = [ + "bitflags", + "crc32fast", + "deflate", + "miniz_oxide", +] + [[package]] name = "pollster" version = "0.2.5" @@ -1849,6 +2027,22 @@ dependencies = [ "thiserror", ] +[[package]] +name = "quanta" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" +dependencies = [ + "crossbeam-utils", + "libc", + "mach", + "once_cell", + "raw-cpuid", + "wasi 0.10.0+wasi-snapshot-preview1", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1875,6 +2069,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", + "rand_pcg", ] [[package]] @@ -1935,6 +2130,24 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + +[[package]] +name = "raw-cpuid" +version = "10.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6aa2540135b6a94f74c7bc90ad4b794f822026a894f3d7bcd185c100d13d4ad6" +dependencies = [ + "bitflags", +] + [[package]] name = "rayon" version = "1.5.3" @@ -2175,6 +2388,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" +[[package]] +name = "scoped_threadpool" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8" + [[package]] name = "scopeguard" version = "1.1.0" @@ -2591,6 +2810,26 @@ dependencies = [ "once_cell", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "tiff" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7259662e32d1e219321eb309d5f9d898b779769d81b76e762c07c8e5d38fcb65" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.1.44" @@ -3214,6 +3453,12 @@ dependencies = [ "webpki 0.22.0", ] +[[package]] +name = "weezl" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9193164d4de03a926d909d3bc7c30543cecb35400c02114792c2cae20d5e2dbb" + [[package]] name = "winapi" version = "0.2.8" diff --git a/dim/src/external/mock.rs b/dim/src/external/mock.rs index e67ce398d..46a7fa50e 100644 --- a/dim/src/external/mock.rs +++ b/dim/src/external/mock.rs @@ -19,7 +19,7 @@ impl ExternalQuery for MockProvider { unimplemented!() } - async fn actors(&self, external_id: &str) -> Result> { + async fn cast(&self, external_id: &str) -> Result> { unimplemented!() } } diff --git a/dim/src/lib.rs b/dim/src/lib.rs index 0c297225d..f46cfdb62 100644 --- a/dim/src/lib.rs +++ b/dim/src/lib.rs @@ -40,8 +40,6 @@ pub mod logger; pub mod routes; /// New generation scanner infrastructure. pub mod scanner; -/// Contains our media scanners and so on. -pub mod scanners; /// Contains the fairing which tracks streams across rest api pub mod stream_tracking; /// Contains all the logic needed for streaming and on-the-fly transcoding. diff --git a/dim/src/scanners/scanner_daemon.rs b/dim/src/scanner/daemon.rs similarity index 100% rename from dim/src/scanners/scanner_daemon.rs rename to dim/src/scanner/daemon.rs diff --git a/dim/src/scanners/base.rs b/dim/src/scanners/base.rs deleted file mode 100644 index dc01941f7..000000000 --- a/dim/src/scanners/base.rs +++ /dev/null @@ -1,472 +0,0 @@ -use displaydoc::Display; -use std::path::Path; -use std::path::PathBuf; -use thiserror::Error; - -use tracing::debug; -use tracing::debug_span; -use tracing::error; -use tracing::info; -use tracing::instrument; -use tracing::warn; -use tracing::Instrument; - -use database::library::MediaType; -use database::mediafile::InsertableMediaFile; -use database::mediafile::MediaFile; -use database::mediafile::UpdateMediaFile; -use database::DbConnection; - -use crate::core::EventTx; -use crate::scanners::movie::MovieMatcher; -use crate::scanners::tmdb::Tmdb; -use crate::scanners::tv_show::TvShowMatcher; -use crate::streaming::ffprobe::FFProbeCtx; -use crate::streaming::FFPROBE_BIN; - -use super::ApiMedia; - -use torrent_name_parser::Metadata; - -use serde::Serialize; - -use tokio::task::spawn_blocking; - -use async_trait::async_trait; -use xtra_proc::actor; -use xtra_proc::handler; - -use anitomy::Anitomy; -use anitomy::ElementCategory; -use anitomy::Elements; - -#[derive(Display, Debug, Error, Serialize, Clone)] -pub enum ScannerError { - /// Could not get a connection to the db - DatabaseConnectionError, - /// The filename parser returned no useful results - FilenameParserError, - /// Something happened to ffprobe - FFProbeError, - /// An unknown error has occured - UnknownError, - /// Database error: {0} - DatabaseError(String), -} - -impl From for ScannerError { - fn from(e: database::DatabaseError) -> Self { - match e { - database::DatabaseError::DatabaseError(e) => Self::DatabaseError(e.to_string()), - } - } -} - -/// `MetadataExtractor` is an actor that processes files on the local filesystem. It parses the -/// filename to extract basic information such as title, year, episode/season. This actor will also -/// run ffprobe on the files to extract other metadata like format and codec. -/// -/// Once a file is parsed and inserted into the database, it is sent to a `MetadataMatcher` actor. -/// Which will query extra external metadata from various APIs. -#[actor] -pub struct MetadataExtractor { - pub conn: database::DbConnection, -} - -#[actor] -impl MetadataExtractor { - #[allow(clippy::new_without_default)] - pub fn new() -> Self { - Self { - conn: database::try_get_conn().unwrap().clone(), - } - } - - #[handler] - #[instrument(skip(self, library_id, _media_type))] - pub async fn mount_file( - &mut self, - file: PathBuf, - library_id: i64, - _media_type: MediaType, - ) -> Result { - let target_file = file.to_str().unwrap().to_owned(); - - let _file_name = if let Some(file_name) = file.file_name().and_then(|x| x.to_str()) { - file_name - } else { - warn!("Received non-unicode filename {}", file = target_file); - - return Err(ScannerError::UnknownError); - }; - - let target_file_clone = target_file.clone(); - let res = { - let mut tx = self - .conn - .read() - .begin() - .await - .map_err(|e| ScannerError::DatabaseError(format!("{:?}", e)))?; - - MediaFile::get_by_file(&mut tx, &target_file_clone).await - }; - - if let Ok(_media_file) = res { - debug!( - file = ?file.to_string_lossy(), - library_id = library_id, - "File already exists in the db", - ); - - return Err(ScannerError::UnknownError); - } - - // we clone so that we can strip the extension. - let mut file_name_clone = file.to_owned(); - file_name_clone.set_extension(""); - // unwrap will never panic because we validate the path earlier on. - let file_name_clone = file_name_clone - .file_name() - .unwrap() - .to_str() - .unwrap() - .to_owned(); - - let clone = file_name_clone.clone().replace(|c: char| !c.is_ascii(), ""); - - // closure needs to be bound because of a lifetime bug where the closure passed to - // `spawn_blocking` lives more than the data moved into it thus we cant pass a reference to - // `Metadata::from` directly. - let meta_from_string = - move || Metadata::from(&clone).map_err(|_| ScannerError::FilenameParserError); - - let metadata = match spawn_blocking(meta_from_string) - .instrument(debug_span!("ParseFilename")) - .await - { - Ok(x) => x?, - Err(e) => { - error!(e = ?e, "Metadata::from possibly panicked"); - return Err(ScannerError::UnknownError); - } - }; - - let file_clone = file.clone(); - let ffprobe_data = if let Ok(data) = FFProbeCtx::new(&FFPROBE_BIN) - .get_meta(file_clone.to_str().unwrap()) - .await - { - data - } else { - error!( - file = ?file.to_string_lossy(), - "Couldnt extract media information with ffprobe", - ); - return Err(ScannerError::FFProbeError); - }; - - let media_file = InsertableMediaFile { - library_id, - media_id: None, - target_file: target_file.to_string(), - - raw_name: metadata.title().to_owned(), - raw_year: metadata.year().map(|x| x as i64), - season: metadata.season().map(|x| x as i64), - episode: metadata.episode().map(|x| x as i64), - - quality: ffprobe_data.get_height().map(|x| x.to_string()), - codec: ffprobe_data.get_video_codec(), - container: ffprobe_data.get_container(), - audio: ffprobe_data - .get_primary_codec("audio") - .map(ToOwned::to_owned), - original_resolution: Default::default(), - duration: ffprobe_data.get_duration().map(|x| x as i64), - corrupt: ffprobe_data.is_corrupt(), - channels: ffprobe_data.get_primary_channels(), - profile: ffprobe_data.get_video_profile(), - audio_language: ffprobe_data - .get_audio_lang() - .or_else(|| ffprobe_data.get_video_lang()) - .as_deref() - .and_then(crate::utils::lang_from_iso639) - .map(ToString::to_string), - }; - - let mediafile = { - let mut lock = self.conn.writer().lock_owned().await; - let mut tx = database::write_tx(&mut lock) - .await - .map_err(|e| ScannerError::DatabaseError(format!("{:?}", e)))?; - - let file_id = media_file - .insert(&mut tx) - .instrument(debug_span!("media_file_insert")) - .await?; - - let mediafile = MediaFile::get_one(&mut tx, file_id) - .instrument(debug_span!("media_file_select")) - .await?; - - assert!(file_id == mediafile.id); - - tx.commit() - .instrument(debug_span!("TxCommit")) - .await - .map_err(|e| ScannerError::DatabaseError(format!("{:?}", e)))?; - drop(lock); - - mediafile - }; - - info!( - file = ?&target_file, - library_id = library_id, - id = mediafile.id, - season = metadata.season().unwrap_or(0), - episode = metadata.episode().unwrap_or(0), - ); - - Ok(mediafile) - } -} - -#[actor] -pub struct MetadataMatcher { - pub movie_tmdb: Tmdb, - pub tv_tmdb: Tmdb, - pub conn: DbConnection, - pub event_tx: EventTx, -} - -#[actor] -impl MetadataMatcher { - pub fn new(conn: DbConnection, event_tx: EventTx) -> Self { - Self { - conn, - event_tx, - movie_tmdb: Tmdb::new("38c372f5bc572c8aadde7a802638534e".into(), MediaType::Movie), - tv_tmdb: Tmdb::new("38c372f5bc572c8aadde7a802638534e".into(), MediaType::Tv), - } - } - - #[handler] - pub async fn match_movie(&mut self, media: MediaFile) -> Result<(), ScannerError> { - let result = match self - .movie_tmdb - .search(media.raw_name.clone(), media.raw_year.map(|x| x as i32)) - .await - { - Ok(v) => v, - Err(e) => { - error!(media = ?media, reason = ?e, "Could not match movie to tmdb"); - - return Err(ScannerError::UnknownError); - } - }; - - self.match_movie_to_result(media, result).await - } - - #[handler] - pub async fn match_movie_to_result( - &mut self, - media: MediaFile, - result: ApiMedia, - ) -> Result<(), ScannerError> { - let matcher = MovieMatcher { - conn: &self.conn, - event_tx: &self.event_tx, - }; - - matcher.match_to_result(result, &media).await; - Ok(()) - } - - #[handler] - pub async fn match_tv(&mut self, media: MediaFile) -> Result<(), ScannerError> { - let mut media = media; - - let path = Path::new(&media.target_file); - let filename = path - .file_name() - .and_then(|x| x.to_str()) - .map(ToString::to_string) - .unwrap_or_default(); - - // FIXME: We an use into_ok_or_err here once it hits stable. - let els: Elements = match spawn_blocking(move || { - let mut anitomy = Anitomy::new(); - anitomy.parse(filename.as_str()) - }) - .await - .unwrap() - { - Ok(v) | Err(v) => v, - }; - - let mut result = self - .tv_tmdb - .search(media.raw_name.clone(), media.raw_year.map(|x| x as i32)) - .await; - - if let Some(x) = els.get(ElementCategory::AnimeTitle) { - if result.is_err() { - // NOTE: If we got here then we assume that the file uses common anime release naming schemes. - // Thus we prioritise metadata extracted by anitomy. - result = self.tv_tmdb.search(x.to_string(), None).await; - - // NOTE: Some releases dont include season number, so we just assume its the first one. - let anitomy_episode = els - .get(ElementCategory::EpisodeNumber) - .and_then(|x| x.parse::().ok()) - .or(media.episode); - - let anitomy_season = els - .get(ElementCategory::AnimeSeason) - .and_then(|x| x.parse::().ok()) - .or(Some(1)); - - let mut lock = self.conn.writer().lock_owned().await; - let mut tx = database::write_tx(&mut lock) - .await - .map_err(|e| ScannerError::DatabaseError(format!("{:?}", e)))?; - - let update_mediafile = UpdateMediaFile { - episode: anitomy_episode.map(|x| x as i64), - season: anitomy_season.map(|x| x as i64), - raw_name: Some(x.to_string()), - ..Default::default() - }; - - let _ = update_mediafile.update(&mut tx, media.id).await; - - tx.commit() - .await - .map_err(|e| ScannerError::DatabaseError(format!("{:?}", e)))?; - - media.episode = anitomy_episode.map(|x| x as i64); - media.season = anitomy_season.map(|x| x as i64); - } - } - - let result = match result { - Ok(v) => v, - Err(e) => { - error!(media = ?media, reason = ?e, "Could not match tv show to tmdb"); - return Err(ScannerError::UnknownError); - } - }; - - self.match_tv_to_result(media, result).await - } - - #[handler] - pub async fn match_tv_to_result( - &mut self, - media: MediaFile, - result: ApiMedia, - ) -> Result<(), ScannerError> { - // FIXME: Our handler macro cant handle `mut` keyword yet. - let mut media = media; - let mut result = result; - - let mut lock = self.conn.writer().lock_owned().await; - let mut tx = database::write_tx(&mut lock) - .await - .map_err(|e| ScannerError::DatabaseError(format!("{:?}", e)))?; - - patch_tv_metadata(&mut media, &mut tx).await?; - - tx.commit() - .await - .map_err(|e| ScannerError::DatabaseError(format!("{:?}", e)))?; - drop(lock); - - let mut seasons: Vec = self - .tv_tmdb - .get_seasons_for(result.id) - .await - .unwrap_or_default() - .into_iter() - .map(Into::into) - .collect(); - - for season in seasons.iter_mut() { - season.episodes = self - .tv_tmdb - .get_episodes_for(result.id, season.season_number) - .await - .unwrap_or_default() - .into_iter() - .map(Into::into) - .collect(); - } - - result.seasons = seasons; - - let matcher = TvShowMatcher { - conn: &self.conn, - event_tx: &self.event_tx, - }; - - matcher.match_to_result(result, &media).await; - Ok(()) - } -} - -#[instrument(skip(media, tx))] -pub async fn patch_tv_metadata( - media: &mut MediaFile, - tx: &mut database::Transaction<'_>, -) -> Result<(), ScannerError> { - // This function is somewhat of a hack. `torrent-name-parser` parses shows and movie names - // well, but it fails to parse anime filenames sometimes, and when it fails it outputs random - // data, thus here we run a 2nd pass metadata parse with anitomy, and if anitomy parses - // everything well we use its episode and season. - let path = Path::new(&media.target_file); - let filename = path - .file_name() - .and_then(|x| x.to_str()) - .map(ToString::to_string) - .unwrap_or_default(); - - // FIXME: Use into_ok_or_err when it hits stable. - let els: Elements = match spawn_blocking(move || { - let mut anitomy = Anitomy::new(); - anitomy.parse(filename.as_str()) - }) - .await - .unwrap() - { - Ok(v) => v, - Err(_) => { - debug!(media = ?media, "patch_tv_metadata exited early"); - return Ok(()); - } - }; - - if let Some(episode) = els - .get(ElementCategory::EpisodeNumber) - .and_then(|x| x.parse::().ok()) - { - let season = els - .get(ElementCategory::AnimeSeason) - .and_then(|x| x.parse::().ok()) - .or(Some(1)); - - let updated_mediafile = UpdateMediaFile { - episode: Some(episode), - season, - ..Default::default() - }; - - let _ = updated_mediafile.update(&mut *tx, media.id).await; - media.episode = Some(episode); - media.season = season; - } - - Ok(()) -} diff --git a/dim/src/scanners/mod.rs b/dim/src/scanners/mod.rs deleted file mode 100644 index 8e7f67222..000000000 --- a/dim/src/scanners/mod.rs +++ /dev/null @@ -1,251 +0,0 @@ -pub mod base; -pub mod movie; -pub mod scanner_daemon; -pub mod tmdb; -pub mod tv_show; - -use chrono::Datelike; -use chrono::NaiveDate; - -use database::library::Library; -use database::library::MediaType; - -use tracing::info; -use tracing::instrument; - -use crate::core::DbConnection; -use crate::core::EventTx; -use crate::json; -use crate::utils::secs_to_pretty; - -use once_cell::sync::OnceCell; -use walkdir::WalkDir; - -use std::path::Path; -use std::path::PathBuf; -use std::time::Instant; - -use serde::Deserialize; -use serde::Serialize; - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ApiMedia { - pub id: u64, - pub title: String, - pub release_date: Option, - pub overview: Option, - pub poster_path: Option, - pub backdrop_path: Option, - pub poster_file: Option, - pub backdrop_file: Option, - pub genres: Vec, - pub rating: Option, - pub seasons: Vec, - pub duration: Option, -} - -impl ApiMedia { - /// Parses a release date string and returns the year. - pub fn year(&self) -> Option { - NaiveDate::parse_from_str(self.release_date.as_ref()?, "%Y-%m-%d") - .ok() - .map(|x| x.year() as _) - } - - /// Converts a collections of `ApiMedia` objects into a REST-API compatible response. - pub fn search_response(items: impl Iterator>) -> warp::reply::Json { - warp::reply::json( - &items - .map(Into::into) - .map(|x| { - json!({ - "id": x.id, - "title": x.title, - "year": x.year(), - "overview": x.overview, - "poster_path": x.poster_path, - "genres": x.genres, - "rating": x.rating, - "duration": x.duration.map(secs_to_pretty), - }) - }) - .collect::>(), - ) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ApiSeason { - pub id: u64, - pub name: Option, - pub poster_path: Option, - pub poster_file: Option, - pub season_number: u64, - pub episodes: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ApiEpisode { - pub id: u64, - pub name: Option, - pub overview: Option, - pub episode: Option, - pub still: Option, - pub still_file: Option, -} - -pub(super) static METADATA_EXTRACTOR: OnceCell = OnceCell::new(); -pub(super) static METADATA_MATCHER: OnceCell = OnceCell::new(); -pub(super) static SUPPORTED_EXTS: &[&str] = &["mp4", "mkv", "avi", "webm"]; - -pub fn get_extractor(_tx: &EventTx) -> &'static base::MetadataExtractor { - let mut handle = xtra::spawn::Tokio::Global; - - METADATA_EXTRACTOR.get_or_init(|| base::MetadataExtractor::cluster(&mut handle, 4).1) -} - -pub fn get_matcher(tx: &EventTx) -> &'static base::MetadataMatcher { - let mut handle = xtra::spawn::Tokio::Global; - - METADATA_MATCHER.get_or_init(|| { - let conn = database::try_get_conn().expect("Failed to grab a connection"); - base::MetadataMatcher::cluster(&mut handle, 6, conn.clone(), tx.clone()).1 - }) -} - -pub fn get_matcher_unchecked() -> &'static base::MetadataMatcher { - METADATA_MATCHER.get().unwrap() -} - -#[doc(hidden)] -pub async fn get_subfiles( - paths: impl Iterator>, -) -> Result, self::base::ScannerError> { - let mut files = Vec::with_capacity(2048); - for path in paths { - let mut subfiles: Vec = WalkDir::new(path) - // we want to follow all symlinks in case of complex dir structures - .follow_links(true) - .into_iter() - .filter_map(Result::ok) - // ignore all hidden files. - .filter(|f| { - !f.path() - .iter() - .any(|s| s.to_str().map(|x| x.starts_with('.')).unwrap_or(false)) - }) - // check whether `f` has a supported extension - .filter(|f| { - f.path() - .extension() - .and_then(|e| e.to_str()) - .map_or(false, |e| SUPPORTED_EXTS.contains(&e)) - }) - .map(|f| f.into_path()) - .collect(); - - files.append(&mut subfiles); - } - - Ok(files) -} - -#[instrument(skip(tx, paths))] -pub async fn start_custom( - library_id: i64, - tx: EventTx, - paths: I, - media_type: MediaType, -) -> Result<(), self::base::ScannerError> -where - I: Iterator, - T: AsRef, -{ - info!(library_id = library_id, "Scanning library"); - - tx.send( - events::Message { - id: library_id, - event_type: events::PushEventType::EventStartedScanning, - } - .to_string(), - ) - .unwrap(); - - let extractor = get_extractor(&tx); - let matcher = get_matcher(&tx); - - let files = get_subfiles(paths).await?; - - let total_files = files.len(); - - info!( - library_id = library_id, - files = total_files, - "Walked library directory", - ); - - let now = Instant::now(); - let mut futures = Vec::new(); - - for file in files { - futures.push(async move { - if let Ok(mfile) = extractor - .mount_file(file.clone(), library_id, media_type) - .await - { - match media_type { - MediaType::Movie => { - let _ = matcher.match_movie(mfile).await; - } - MediaType::Tv => { - let _ = matcher.match_tv(mfile).await; - } - _ => unreachable!(), - } - } - }) - } - - futures::future::join_all(futures).await; - - info!( - library_id = library_id, - files = total_files, - duration = now.elapsed().as_secs(), - "Finished scanning library", - ); - - tx.send( - events::Message { - id: library_id, - event_type: events::PushEventType::EventStoppedScanning, - } - .to_string(), - ) - .unwrap(); - - Ok(()) -} - -pub async fn start( - conn: DbConnection, - id: i64, - tx: EventTx, -) -> Result<(), self::base::ScannerError> { - let mut tx_ = conn - .read() - .begin() - .await - .map_err(|e| self::base::ScannerError::DatabaseError(format!("{:?}", e)))?; - - let lib = Library::get_one(&mut tx_, id).await?; - - start_custom(id, tx, lib.locations.into_iter(), lib.media_type).await -} - -/// Function formats the path where assets are stored. -pub fn format_path(x: Option) -> String { - x.map(|x| format!("images/{}", x.trim_start_matches('/'))) - .unwrap_or_default() -} diff --git a/dim/src/scanners/movie.rs b/dim/src/scanners/movie.rs deleted file mode 100644 index 67af3852b..000000000 --- a/dim/src/scanners/movie.rs +++ /dev/null @@ -1,218 +0,0 @@ -use database::asset::InsertableAsset; -use database::genre::InsertableGenre; -use database::genre::InsertableGenreMedia; -use database::DbConnection; - -use database::library::MediaType; -use database::media::InsertableMedia; -use database::mediafile::MediaFile; -use database::mediafile::UpdateMediaFile; - -use chrono::prelude::Utc; -use chrono::Datelike; -use chrono::NaiveDate; - -use events::Message; -use events::PushEventType; - -use tracing::error; -use tracing::instrument; -use tracing::warn; - -use super::format_path; -use crate::core::EventTx; -use crate::fetcher::insert_into_queue; - -pub struct MovieMatcher<'a> { - pub conn: &'a DbConnection, - pub event_tx: &'a EventTx, -} - -impl<'a> MovieMatcher<'a> { - #[instrument(skip(self, result), fields(result.id = %result.id, result.name = %result.title))] - pub async fn match_to_result(&self, result: super::ApiMedia, orphan: &'a MediaFile) { - let library_id = orphan.library_id; - - let mut lock = self.conn.writer().lock_owned().await; - let mut tx = match database::write_tx(&mut lock).await { - Ok(x) => x, - Err(e) => { - error!(reason = ?e, "Failed to create transaction."); - return; - } - }; - - let media_id = match self.inner_match(result, orphan, &mut tx, None).await { - Ok(x) => x, - Err(e) => { - error!(reason = ?e, "Failed to match media"); - return; - } - }; - - if let Err(e) = tx.commit().await { - error!(reason = ?e, "Failed to commit transaction."); - return; - } - - self.push_event(media_id, library_id, orphan.id).await; - } - - pub async fn inner_match( - &self, - result: super::ApiMedia, - orphan: &'a MediaFile, - tx: &mut database::Transaction<'_>, - reuse_media_id: Option, - ) -> Result { - let name = result.title.clone(); - - let year: Option = result - .release_date - .as_ref() - .cloned() - .map(|x| NaiveDate::parse_from_str(x.as_str(), "%Y-%m-%d")) - .map(Result::ok) - .unwrap_or(None) - .map(|s| s.year() as i64); - - let poster_path = result.poster_path.clone(); - - let backdrop_path = result.backdrop_path.clone(); - - if let Some(poster_path) = poster_path.as_ref() { - let _ = insert_into_queue(poster_path.clone(), 3); - } - - if let Some(backdrop_path) = backdrop_path.as_ref() { - let _ = insert_into_queue(backdrop_path.clone(), 3); - } - - let poster = match poster_path { - Some(path) => { - let asset = InsertableAsset { - remote_url: Some(path), - local_path: format_path(result.poster_file.clone()), - file_ext: "jpg".into(), - } - .insert(&mut *tx) - .await; - - match asset { - Ok(x) => Some(x.id), - Err(e) => { - warn!( - reason = ?e, - orphan_id = orphan.id, - "Failed to insert poster into db", - ); - None - } - } - } - None => None, - }; - - let backdrop = match backdrop_path { - Some(path) => { - let asset = InsertableAsset { - remote_url: Some(path), - local_path: format_path(result.backdrop_file.clone()), - file_ext: "jpg".into(), - } - .insert(&mut *tx) - .await; - - match asset { - Ok(x) => Some(x.id), - Err(e) => { - warn!( - reason = ?e, - orphan_id = orphan.id, - "Failed to insert backdrop into db", - ); - None - } - } - } - None => None, - }; - - let media = InsertableMedia { - library_id: orphan.library_id, - name, - description: result.overview.clone(), - rating: result.rating, - year, - added: Utc::now().to_string(), - - poster, - backdrop, - media_type: MediaType::Movie, - }; - - let media_id = self - .inner_insert(orphan, media, result, &mut *tx, reuse_media_id) - .await - .map_err(|e| { - error!(reason = ?e, orphan_id = orphan.id, "Failed to insert new media."); - e - })?; - - Ok(media_id) - } - - async fn inner_insert( - &self, - orphan: &MediaFile, - media: InsertableMedia, - result: super::ApiMedia, - tx: &mut database::Transaction<'_>, - reuse_media_id: Option, - ) -> Result { - let media_id = if let Some(id) = reuse_media_id { - media.insert_with_id(&mut *tx, id).await? - } else { - media.insert(&mut *tx).await? - }; - - for name in result.genres { - let genre = InsertableGenre { name }; - - if let Ok(x) = genre.insert(&mut *tx).await { - let _ = InsertableGenreMedia::insert_pair(x, media_id, &mut *tx).await; - } - } - - let updated_mediafile = UpdateMediaFile { - media_id: Some(media_id), - ..Default::default() - }; - - updated_mediafile.update(&mut *tx, orphan.id).await?; - - Ok(media_id) - } - - async fn push_event(&self, id: i64, lib_id: i64, mediafile: i64) { - // TODO: verify if this scanner suffers from the same duplicate top-level media insertion - // bug. - let event = Message { - id, - event_type: PushEventType::EventNewCard { lib_id }, - }; - - let _ = self.event_tx.send(serde_json::to_string(&event).unwrap()); - - // Notify that a mediafile was matched. - let event = Message { - id, - event_type: PushEventType::MediafileMatched { - mediafile, - library_id: lib_id, - }, - }; - - let _ = self.event_tx.send(serde_json::to_string(&event).unwrap()); - } -} diff --git a/dim/src/scanners/tmdb.rs b/dim/src/scanners/tmdb.rs deleted file mode 100644 index 6f3220a05..000000000 --- a/dim/src/scanners/tmdb.rs +++ /dev/null @@ -1,576 +0,0 @@ -pub(crate) use database::library::MediaType; -use serde::Deserialize; -use serde::Serialize; - -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use reqwest::Client; -use reqwest::ClientBuilder; -use reqwest::StatusCode; - -use displaydoc::Display; -use futures::stream; -use futures::StreamExt; -use thiserror::Error; -use tokio::sync::RwLock; - -use async_recursion::async_recursion; - -static APP_USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "/", env!("CARGO_PKG_VERSION"),); - -#[derive(Clone, Display, Debug, Error, Serialize)] -pub enum TmdbError { - /// The request timeouted - Timeout, - /// Max retry count reached - ReachedMaxTries, - /// Internal error with reqwest - ReqwestError, - /// The json returned could not be deserialized: {0:?} - DeserializationError(String), - /// No results are found: query={query} year={year:?} - NoResults { query: String, year: Option }, - /// No seasons found for the id supplied: {id} - NoSeasonsFound { id: u64 }, - /// No episodes found for the id supplied: id={id} season={season} - NoEpisodesFound { id: u64, season: u64 }, - /// Could not find genre with supplied id: {id} - NoGenreFound { id: u64 }, - /// Failed to search for id {id} in tmdb: {response:?} - SearchByIdNotFound { id: i32, response: ServerError }, - /// Failed to deserialize server error: {0} - ErrorDeserializationError(String), -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct ServerError { - status_message: String, - status_code: i32, -} - -#[derive(Clone)] -pub struct Tmdb { - api_key: String, - client: Client, - base: String, - media_type: MediaType, -} - -impl Tmdb { - pub fn new(api_key: String, media_type: MediaType) -> Self { - let client = ClientBuilder::new().user_agent(APP_USER_AGENT); - - Self { - api_key, - client: client.build().unwrap(), - base: "https://api.themoviedb.org/3".into(), - media_type, - } - } - - pub async fn search( - &mut self, - title: String, - year: Option, - ) -> Result { - self.search_by_name(title.clone(), year, None) - .await? - .first() - .cloned() - .map(Into::into) - .ok_or(TmdbError::NoResults { query: title, year }) - } - - pub async fn search_by_id(&mut self, id: i32) -> Result { - let args = vec![ - ("api_key".to_string(), self.api_key.clone()), - ("language".to_string(), "en-US".into()), - ]; - - let url = format!("{}/{}/{}", self.base, self.media_type, id); - let req = self - .client - .get(url) - .query(&args) - .send() - .await - .map_err(|_| TmdbError::ReqwestError)?; - - #[derive(Deserialize, Clone, Debug)] - struct WMedia { - pub id: u64, - #[serde(rename(deserialize = "original_title", deserialize = "original_name"))] - pub title: String, - #[serde(rename(deserialize = "release_date", deserialize = "first_air_date"))] - pub release_date: Option, - pub overview: Option, - pub vote_average: Option, - pub poster_path: Option, - pub backdrop_path: Option, - pub genres: Vec, - pub runtime: Option, - } - - #[derive(Deserialize, Clone, Debug)] - struct GenrePair { - #[allow(dead_code)] - pub id: u64, - pub name: String, - } - - if !req.status().is_success() { - return Err(TmdbError::SearchByIdNotFound { - id, - response: req - .json::() - .await - .map_err(|e| TmdbError::ErrorDeserializationError(e.to_string()))?, - }); - } - - let result: WMedia = req - .json::() - .await - .map_err(|e| TmdbError::DeserializationError(e.to_string()))?; - - Ok(Media { - id: result.id, - title: result.title, - release_date: result.release_date, - overview: result.overview, - vote_average: result.vote_average, - poster_path: result.poster_path, - backdrop_path: result.backdrop_path, - genre_ids: None, - genres: result - .genres - .into_iter() - .map(|x| x.name) - .collect::>(), - runtime: result.runtime, - }) - } - - #[async_recursion] - pub async fn search_by_name( - &mut self, - title: String, - year: Option, - max_tries: Option, - ) -> Result, TmdbError> { - type CacheKey = (String, Option, MediaType); - type CacheStore = Arc>>>; - - lazy_static::lazy_static! { - static ref __CACHE: CacheStore = Arc::new(RwLock::new(HashMap::new())); - } - - { - let lock = (*__CACHE).read().await; - let key = (title.clone(), year, self.media_type); - - if let Some(x) = lock.get(&key) { - return Ok(x.to_vec()); - } - } - - let max_tries = max_tries.unwrap_or(10); - - if max_tries == 0 { - return Err(TmdbError::ReachedMaxTries); - } - - let mut args: Vec<(String, String)> = vec![ - ("api_key".to_string(), self.api_key.clone()), - ("language".to_string(), "en-US".into()), - ("query".to_string(), title.clone()), - ("page".to_string(), "1".into()), - ("include_adult".to_string(), "false".into()), - ]; - - if let Some(year) = year { - args.push(("year".into(), year.to_string())); - } - - let url = format!("{}/search/{}", self.base, self.media_type); - - let req = self - .client - .get(url) - .query(&args) - .send() - .await - .map_err(|_| TmdbError::ReqwestError)?; - - if matches!(req.status(), StatusCode::TOO_MANY_REQUESTS) { - tokio::time::sleep(Duration::from_millis(1000)).await; - return self.search_by_name(title, year, Some(max_tries - 1)).await; - } - - let mut result: Vec = req - .json::() - .await - .map_err(|e| TmdbError::DeserializationError(e.to_string()))? - .results - .into_iter() - .flatten() - .collect(); - - for media in result.iter_mut() { - let ids = media.genre_ids.clone().unwrap_or_default(); - media.genres = stream::iter(ids) - .filter_map(|x| { - let client = ClientBuilder::new().user_agent(APP_USER_AGENT); - let mut this = Tmdb { - api_key: self.api_key.clone(), - client: client.build().unwrap(), - base: self.base.clone(), - media_type: self.media_type, - }; - - async move { this.get_genre_detail(x).await.ok().map(|x| x.name) } - }) - .collect::>() - .await; - } - - { - let mut lock = (*__CACHE).write().await; - let key = (title.clone(), year, self.media_type); - lock.insert(key, result.clone()); - } - - Ok(result) - } - - pub async fn get_seasons_for(&mut self, id: u64) -> Result, TmdbError> { - let args = vec![("api_key".to_string(), self.api_key.clone())]; - - let req = self - .client - .get(format!("{}/tv/{}", self.base, id)) - .query(&args) - .send() - .await - .map_err(|_| TmdbError::ReqwestError)?; - - #[derive(Deserialize)] - struct Wrapper { - seasons: Option>, - } - - req.json::() - .await - .map_err(|e| TmdbError::DeserializationError(e.to_string()))? - .seasons - .ok_or(TmdbError::NoSeasonsFound { id }) - } - - pub async fn get_episodes_for( - &mut self, - id: u64, - season: u64, - ) -> Result, TmdbError> { - let args = vec![("api_key".to_string(), self.api_key.clone())]; - - let req = self - .client - .get(format!("{}/tv/{}/season/{}", self.base, id, season)) - .query(&args) - .send() - .await - .map_err(|_| TmdbError::ReqwestError)?; - - #[derive(Deserialize)] - struct Wrapper { - episodes: Option>, - } - - req.json::() - .await - .map_err(|e| TmdbError::DeserializationError(e.to_string()))? - .episodes - .ok_or(TmdbError::NoEpisodesFound { id, season }) - } - - pub async fn get_genre_detail(&mut self, genre_id: u64) -> Result { - lazy_static::lazy_static! { - static ref __CACHE: Arc>>> = Arc::new(RwLock::new(HashMap::new())); - } - - { - let lock = (*__CACHE).read().await; - if let Some(x) = lock.get(&self.media_type) { - if let Some(x) = x.iter().find(|x| x.id == genre_id) { - return Ok(x.clone()); - } - } - } - - let args = vec![("api_key".to_string(), self.api_key.clone())]; - - let url = format!("{}/genre/{}/list", self.base.clone(), self.media_type); - let req = self - .client - .get(url) - .query(&args) - .send() - .await - .map_err(|_| TmdbError::ReqwestError)?; - - #[derive(Deserialize)] - struct Wrapper { - genres: Vec, - } - - let genres = req - .json::() - .await - .map_err(|e| TmdbError::DeserializationError(e.to_string()))? - .genres; - - { - let mut lock = (*__CACHE).write().await; - lock.insert(self.media_type, genres.clone()); - } - - genres - .iter() - .find(|x| x.id == genre_id) - .cloned() - .ok_or(TmdbError::NoGenreFound { id: genre_id }) - } -} -/* - - { - "page": 1, - "results": [ - { - "adult": false, - "backdrop_path": "/1stUIsjawROZxjiCMtqqXqgfZWC.jpg", - "genre_ids": [ - 12, - 14 - ], - "id": 672, - "original_language": "en", - "original_title": "Harry Potter and the Chamber of Secrets", - "overview": "Cars fly, trees fight back, and a mysterious house-elf comes to warn Harry Potter at the start of his second year at Hogwarts. Adventure and danger await when bloody writing on a wall announces: The Chamber Of Secrets Has Been Opened. To save Hogwarts will require all of Harry, Ron and Hermione’s magical abilities and courage.", - "popularity": 118.243, - "poster_path": "/sdEOH0992YZ0QSxgXNIGLq1ToUi.jpg", - "release_date": "2002-11-13", - "title": "Harry Potter and the Chamber of Secrets", - "video": false, - "vote_average": 7.7, - "vote_count": 16310 - } - ], - "total_pages": 1, - "total_results": 1 -} -*/ - -#[derive(Deserialize, Clone)] -struct SearchResult { - results: Vec>, -} - -#[derive(Deserialize, Clone, Debug)] -pub struct Media { - pub id: u64, - #[serde(rename(deserialize = "title", deserialize = "name"))] - pub title: String, - #[serde(rename(deserialize = "release_date", deserialize = "first_air_date"))] - pub release_date: Option, - pub overview: Option, - pub vote_average: Option, - pub poster_path: Option, - pub backdrop_path: Option, - pub genre_ids: Option>, - #[serde(skip_deserializing)] - pub genres: Vec, - pub runtime: Option, -} - -impl From for super::ApiMedia { - fn from(this: Media) -> Self { - let backdrop_path = this.backdrop_path.clone().map(|bp| { - if bp.starts_with('/') { - format!("https://image.tmdb.org/t/p/original/{}", bp) - } else { - format!("https://image.tmdb.org/t/p/original{}", bp) - } - }); - - Self { - id: this.id, - title: this.title, - release_date: this.release_date, - overview: this.overview, - poster_path: this - .poster_path - .clone() - .map(|s| format!("https://image.tmdb.org/t/p/w600_and_h900_bestv2{}", s)), - poster_file: this.poster_path, - backdrop_path, - backdrop_file: this.backdrop_path, - genres: this.genres, - rating: this.vote_average, - seasons: Vec::new(), - duration: this.runtime, - } - } -} - -#[derive(Deserialize, Clone, Debug)] -pub struct Genre { - pub id: u64, - pub name: String, -} - -#[derive(Deserialize, Clone, Debug)] -pub struct Season { - pub id: u64, - pub air_date: Option, - pub episode_count: Option, - pub name: Option, - pub overview: Option, - pub poster_path: Option, - pub season_number: Option, -} - -impl From for super::ApiSeason { - fn from(this: Season) -> Self { - Self { - id: this.id, - name: this.name, - poster_path: this - .poster_path - .clone() - .map(|s| format!("https://image.tmdb.org/t/p/w600_and_h900_bestv2{}", s)), - poster_file: this.poster_path.clone(), - season_number: this.season_number.unwrap_or(1), - episodes: Vec::new(), - } - } -} - -#[derive(Deserialize, Clone, Debug)] -pub struct Episode { - pub id: u64, - pub name: Option, - pub overview: Option, - pub episode_number: Option, - pub still_path: Option, -} - -impl From for super::ApiEpisode { - fn from(other: Episode) -> Self { - Self { - id: other.id, - name: other.name, - overview: other.overview, - episode: other.episode_number, - still: other - .still_path - .clone() - .map(|s| format!("https://image.tmdb.org/t/p/w600_and_h900_bestv2{}", s)), - still_file: other.still_path, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - const API_KEY: &str = "38c372f5bc572c8aadde7a802638534e"; - - // #[test] - // fn test_search_by_name() { - // let mut tmdb = Tmdb::new(API_KEY.to_string(), MediaType::Movie); - // let result = tmdb - // .search_by_name("Blade Runner 2049".into(), None, None) - // .unwrap(); - - // let result = result.first().unwrap(); - // assert_eq!(result.title, "Blade Runner 2049"); - // assert_eq!(result.release_date, Some("2017-10-04".into())); - - // let result = tmdb - // .search_by_name("Blade Runner 2049".into(), Some(2017), None) - // .unwrap(); - - // let result = result.first().unwrap(); - // assert_eq!(result.title, "Blade Runner 2049"); - // assert_eq!(result.release_date, Some("2017-10-04".into())); - // assert!(result.overview.is_some()); - - // let mut tmdb = Tmdb::new(API_KEY.to_string(), MediaType::Tv); - // let result = tmdb - // .search_by_name("The expanse".into(), None, None) - // .unwrap(); - - // let result = result.first().unwrap(); - // assert_eq!(result.title, "The Expanse"); - // assert_eq!(result.release_date, Some("2015-12-14".into())); - // assert!(result.overview.is_some()); - // assert!(result.poster_path.is_some()); - // } - - // #[test] - // fn test_get_seasons_for() { - // let mut tmdb = Tmdb::new(API_KEY.to_string(), MediaType::Tv); - // let result = tmdb - // .search_by_name("The expanse".into(), None, None) - // .unwrap(); - - // let result = result.first().unwrap(); - // let seasons = tmdb.get_seasons_for(&result).unwrap(); - - // assert_eq!(seasons.len(), 6); - - // let season1 = &seasons[1]; - // assert_eq!(season1.air_date, Some("2015-12-14".into())); - // assert_eq!(season1.episode_count, Some(10)); - // assert_eq!(season1.season_number, Some(1)); - // assert_eq!(season1.name, Some("Season 1".into())); - // assert!(season1.overview.is_some()); - // } - - // #[test] - // fn test_get_episodes_for() { - // let mut tmdb = Tmdb::new(API_KEY.to_string(), MediaType::Tv); - // let result = tmdb - // .search_by_name("The expanse".into(), None, None) - // .unwrap(); - - // let result = result.first().unwrap(); - // let seasons = tmdb.get_seasons_for(&result).unwrap(); - - // assert_eq!(seasons.len(), 6); - - // let season1 = &seasons[1]; - - // let result = tmdb - // .get_episodes_for(&result, season1.season_number.unwrap()) - // .unwrap(); - // assert_eq!(result.len(), 10); - // } - - // #[test] - // fn test_get_genre_detail() { - // let mut tmdb = Tmdb::new(API_KEY.to_string(), MediaType::Tv); - // let result = tmdb - // .search_by_name("The expanse".into(), None, None) - // .unwrap(); - - // let genres = result.first().unwrap().genre_ids.as_ref().unwrap(); - - // let result = tmdb.get_genre_detail(genres[0]).unwrap(); - // assert_eq!(result.name, "Drama".to_string()); - // } -} diff --git a/dim/src/scanners/tv_show.rs b/dim/src/scanners/tv_show.rs deleted file mode 100644 index 476029c49..000000000 --- a/dim/src/scanners/tv_show.rs +++ /dev/null @@ -1,366 +0,0 @@ -use database::asset::InsertableAsset; -use database::genre::InsertableGenre; -use database::genre::InsertableGenreMedia; -use database::DbConnection; - -use database::episode::InsertableEpisode; -use database::library::MediaType; -use database::media::InsertableMedia; -use database::mediafile::MediaFile; -use database::mediafile::UpdateMediaFile; -use database::season::InsertableSeason; - -use chrono::prelude::Utc; -use chrono::Datelike; -use chrono::NaiveDate; - -use events::Message; -use events::PushEventType; - -use tracing::debug; -use tracing::debug_span; -use tracing::error; -use tracing::instrument; -use tracing::warn; -use tracing::Instrument; - -use super::format_path; -use crate::core::EventTx; -use crate::fetcher::insert_into_queue; - -pub struct TvShowMatcher<'a> { - pub conn: &'a DbConnection, - pub event_tx: &'a EventTx, -} - -impl<'a> TvShowMatcher<'a> { - #[instrument(skip(self, result, orphan), fields(result.id = %result.id, result.name = %result.title, orphan.id = %orphan.id))] - pub async fn match_to_result(&self, result: super::ApiMedia, orphan: &'a MediaFile) { - let library_id = orphan.library_id; - let mut lock = self.conn.writer().lock_owned().await; - let mut tx = match database::write_tx(&mut lock).await { - Ok(x) => x, - Err(e) => { - error!(reason = ?e, "Failed to create transaction."); - return; - } - }; - - let media_id = match self.inner_match(result, orphan, &mut tx, None).await { - Ok(x) => x, - Err(e) => { - error!(reason = ?e, "Failed to match media"); - return; - } - }; - - if let Err(e) = tx.commit().instrument(debug_span!("TxCommit")).await { - error!(reason = ?e, "Failed to commit transaction."); - return; - } - - self.push_event(media_id, library_id, orphan.id).await; - } - - pub async fn inner_match( - &self, - result: super::ApiMedia, - orphan: &'a MediaFile, - tx: &mut database::Transaction<'_>, - reuse_media_id: Option, - ) -> Result { - let name = result.title.clone(); - - let year: Option = result - .release_date - .clone() - .map(|x| NaiveDate::parse_from_str(x.as_str(), "%Y-%m-%d")) - .map(Result::ok) - .unwrap_or(None) - .map(|s| s.year() as i64); - - let poster_path = result.poster_path.clone(); - - let backdrop_path = result.backdrop_path.clone(); - - if let Some(poster_path) = poster_path.as_ref() { - let _ = insert_into_queue(poster_path.clone(), 3).await; - } - - if let Some(backdrop_path) = backdrop_path.as_ref() { - let _ = insert_into_queue(backdrop_path.clone(), 3).await; - } - - let poster = match poster_path { - Some(path) => { - let asset = InsertableAsset { - remote_url: Some(path), - local_path: format_path(result.poster_file.clone()), - file_ext: "jpg".into(), - } - .insert(&mut *tx) - .await; - - match asset { - Ok(x) => Some(x.id), - Err(e) => { - warn!( - reason = ?e, - orphan_id = orphan.id, - "Failed to insert poster into db", - ); - - None - } - } - } - None => None, - }; - - let backdrop = match backdrop_path { - Some(path) => { - let asset = InsertableAsset { - remote_url: Some(path), - local_path: format_path(result.backdrop_file.clone()), - file_ext: "jpg".into(), - } - .insert(&mut *tx) - .await; - - match asset { - Ok(x) => Some(x.id), - Err(e) => { - warn!( - reason = ?e, - orphan_id = orphan.id, - "Failed to insert backdrop into db", - ); - None - } - } - } - None => None, - }; - - let media = InsertableMedia { - name, - year, - library_id: orphan.library_id, - description: result.overview.clone(), - rating: result.rating, - added: Utc::now().to_string(), - poster, - backdrop, - media_type: MediaType::Tv, - }; - - let media_id = self - .inner_insert(orphan, media, result, &mut *tx, reuse_media_id) - .await - .map_err(|e| { - error!(reason = ?e, orphan_id = orphan.id, "Failed to insert new media."); - e - })?; - - Ok(media_id) - } - - #[instrument(skip(self, result, orphan, tx, reuse_media_id), level = "debug")] - async fn inner_insert( - &self, - orphan: &MediaFile, - media: InsertableMedia, - result: super::ApiMedia, - tx: &mut database::Transaction<'_>, - reuse_media_id: Option, - ) -> Result { - let media_id = if let Some(id) = reuse_media_id { - media.insert_with_id(&mut *tx, id).await? - } else { - media.insert(&mut *tx).await? - }; - - for name in result.genres { - let genre = InsertableGenre { name }; - - if let Ok(x) = genre.insert(&mut *tx).await { - let _ = InsertableGenreMedia::insert_pair(x, media_id, &mut *tx).await; - } - } - - let season = { - let orphan_season = orphan.season.unwrap_or(0) as u64; - - result - .seasons - .iter() - .find(|s| s.season_number == orphan_season) - }; - - let poster_file = season.and_then(|x| x.poster_path.clone()); - - if let Some(x) = poster_file.as_ref() { - let _ = insert_into_queue(x.clone(), 2).await; - } - - let season_poster = match poster_file { - Some(path) => { - let asset = InsertableAsset { - remote_url: Some(path), - local_path: format_path(season.and_then(|x| x.poster_file.clone())), - file_ext: "jpg".into(), - } - .insert(&mut *tx) - .await; - - match asset { - Ok(x) => Some(x.id), - Err(e) => { - warn!( - reason = ?e, - orphan_id = orphan.id, - "Failed to insert season poster into db" - ); - - None - } - } - } - None => None, - }; - - let insertable_season = InsertableSeason { - season_number: orphan.season.unwrap_or(0), - added: Utc::now().to_string(), - poster: season_poster, - }; - - let seasonid = match insertable_season.insert(&mut *tx, media_id).await { - Ok(x) => x, - Err(e) => { - warn!( - "Failed to insert season into the database. {}", - reason = e.to_string() - ); - return Err(e.into()); - } - }; - - let search_ep = { - let orphan_episode = orphan.episode.unwrap_or(0) as u64; - season.and_then(|x| { - x.episodes - .iter() - .find(|&s| s.episode == Some(orphan_episode)) - }) - }; - - let still = search_ep.as_ref().and_then(|x| x.still.clone()); - - if let Some(x) = still.as_ref() { - let _ = insert_into_queue(x.clone(), 1).await; - } - - let backdrop = match still { - Some(path) => { - let asset = InsertableAsset { - remote_url: Some(path), - local_path: format_path(search_ep.and_then(|x| x.still_file.clone()).clone()), - file_ext: "jpg".into(), - } - .insert(&mut *tx) - .await; - - match asset { - Ok(x) => Some(x.id), - Err(e) => { - warn!( - reason = ?e, - orphan_id = orphan.id, - "Failed to insert still into db", - ); - - None - } - } - } - None => None, - }; - - debug!( - seasonid = seasonid, - episode = orphan.episode.unwrap_or(0), - target_file = ?&orphan.target_file, - "Inserting new episode", - ); - - let episode = InsertableEpisode { - episode: orphan.episode.unwrap_or(0), - seasonid, - media: InsertableMedia { - library_id: orphan.library_id, - name: search_ep - .as_ref() - .and_then(|x| x.name.clone()) - .unwrap_or_else(|| orphan.episode.unwrap_or(0).to_string()), - added: Utc::now().to_string(), - media_type: MediaType::Episode, - description: search_ep - .as_ref() - .map(|x| x.overview.clone()) - .unwrap_or_default(), - backdrop, - ..Default::default() - }, - }; - - // manually insert the underlying `media` into the table and convert it into a streamable movie/ep - let raw_ep_id = episode.media.insert_blind(&mut *tx).await?; - let episode_id = episode.insert(&mut *tx).await?; - - let updated_mediafile = UpdateMediaFile { - media_id: Some(episode_id), - ..Default::default() - }; - - updated_mediafile - .update(&mut *tx, orphan.id) - .instrument(debug_span!("UpdateMediafile")) - .await?; - - Ok(media_id) - } - - async fn push_event(&self, id: i64, lib_id: i64, mediafile: i64) { - use once_cell::sync::Lazy; - use std::sync::Mutex; - - static DUPLICATE_LOG: Lazy>> = Lazy::new(Default::default); - - // Notify that a mediafile was matched. - let event = Message { - id, - event_type: PushEventType::MediafileMatched { - mediafile, - library_id: lib_id, - }, - }; - - let _ = self.event_tx.send(serde_json::to_string(&event).unwrap()); - - { - let mut lock = DUPLICATE_LOG.lock().unwrap(); - if lock.contains(&(lib_id, id)) { - return; - } - lock.push((lib_id, id)); - } - - let event = Message { - id, - event_type: PushEventType::EventNewCard { lib_id }, - }; - - let _ = self.event_tx.send(serde_json::to_string(&event).unwrap()); - } -}