Metadata group export (#737)

* fix(frontend): use correct verb

* docs: add information about insecure cookies

* feat(backend): add metadata groups to complete export

* feat(backend): allow exporting media groups

* fix(backend): change the exported key name

* feat(backend): allow importing media groups

* fix(backend): change casing of names

* feat: media group json changes

* build(backend): bump version

* refactor(backend): use default params

* feat(database): add total_time_spent column for seen

* feat(backend): return `seen.total_time_spent`

* feat(frontend): display timeSpent for seen

* fix(frontend): format time specifier

* chore(frontend): change order of stats displayed

* chore(frontend): apply formatting

* fix(frontend): use correct param for pagination component

* refactor(frontend): hoist common prop to component

* fix(frontend): change pagination default props

* fix(backend): use starttls for sending email

* chore(ci): remove mailpit from setup

.devcontainer/docker-compose.yml

@@ -33,15 +33,6 @@ services:
       POSTGRES_USER: postgres
       POSTGRES_DB: postgres
 
-  mail:
-    image: axllent/mailpit
-    ports:
-      - 8025:8025
-    environment:
-      MP_SMTP_AUTH_ACCEPT_ANY: "1"
-      MP_SMTP_AUTH_ALLOW_INSECURE: "1"
-      MP_VERBOSE: "1"
-
 volumes:
   minio_storage:
   postgres_storage:

apps/backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ryot"
-version = "4.3.16"
+version = "4.3.17"
 edition = "2021"
 repository = "https://github.com/IgnisDa/ryot"
 license = "GPL-3.0"

apps/backend/src/entities/seen.rs

@@ -43,6 +43,7 @@ pub struct Model {
     // Generated columns
     pub last_updated_on: DateTimeUtc,
     pub num_times_updated: i32,
+    pub total_time_spent: Option<i32>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

apps/backend/src/exporter.rs

@@ -106,6 +106,11 @@ impl ExporterService {
                         .export_media(user_id, &mut writer)
                         .await?;
                 }
+                ExportItem::MediaGroup => {
+                    self.media_service
+                        .export_media_group(user_id, &mut writer)
+                        .await?;
+                }
                 ExportItem::People => {
                     self.media_service
                         .export_people(user_id, &mut writer)

apps/backend/src/importer/audiobookshelf.rs

@@ -121,9 +121,6 @@ pub async fn import(input: DeployAudiobookshelfImportInput) -> Result<ImportResu
     Ok(ImportResult {
         media,
         failed_items,
-        people: vec![],
-        workouts: vec![],
-        collections: vec![],
-        measurements: vec![],
+        ..Default::default()
     })
 }

apps/backend/src/importer/goodreads.rs

@@ -159,9 +159,6 @@ pub async fn import(
     Ok(ImportResult {
         media,
         failed_items,
-        people: vec![],
-        workouts: vec![],
-        collections: vec![],
-        measurements: vec![],
+        ..Default::default()
     })
 }

apps/backend/src/importer/json.rs

@@ -8,7 +8,8 @@ use crate::{
     fitness::resolver::ExerciseService,
     importer::{DeployJsonImportInput, ImportResult},
     models::media::{
-        ImportOrExportItemIdentifier, ImportOrExportMediaItem, ImportOrExportPersonItem,
+        ImportOrExportItemIdentifier, ImportOrExportMediaGroupItem, ImportOrExportMediaItem,
+        ImportOrExportPersonItem,
     },
 };
 
@@ -26,11 +27,7 @@ pub async fn media_import(input: DeployJsonImportInput) -> Result<ImportResult>
     });
     Ok(ImportResult {
         media,
-        people: vec![],
-        workouts: vec![],
-        collections: vec![],
-        failed_items: vec![],
-        measurements: vec![],
+        ..Default::default()
     })
 }
 
@@ -39,11 +36,7 @@ pub async fn measurements_import(input: DeployJsonImportInput) -> Result<ImportR
     let measurements = serde_json::from_str::<Vec<user_measurement::Model>>(&export).unwrap();
     Ok(ImportResult {
         measurements,
-        people: vec![],
-        media: vec![],
-        workouts: vec![],
-        collections: vec![],
-        failed_items: vec![],
+        ..Default::default()
     })
 }
 
@@ -52,11 +45,7 @@ pub async fn people_import(input: DeployJsonImportInput) -> Result<ImportResult>
     let people = serde_json::from_str::<Vec<ImportOrExportPersonItem>>(&export).unwrap();
     Ok(ImportResult {
         people,
-        media: vec![],
-        workouts: vec![],
-        collections: vec![],
-        measurements: vec![],
-        failed_items: vec![],
+        ..Default::default()
     })
 }
 
@@ -72,10 +61,15 @@ pub async fn workouts_import(
         .collect();
     Ok(ImportResult {
         workouts,
-        people: vec![],
-        media: vec![],
-        collections: vec![],
-        measurements: vec![],
-        failed_items: vec![],
+        ..Default::default()
+    })
+}
+
+pub async fn media_groups_import(input: DeployJsonImportInput) -> Result<ImportResult> {
+    let export = fs::read_to_string(input.export)?;
+    let media_groups = serde_json::from_str::<Vec<ImportOrExportMediaGroupItem>>(&export).unwrap();
+    Ok(ImportResult {
+        media_groups,
+        ..Default::default()
     })
 }

apps/backend/src/importer/mal.rs

@@ -90,11 +90,7 @@ pub async fn import(input: DeployMalImportInput) -> Result<ImportResult> {
     }
     Ok(ImportResult {
         media,
-        people: vec![],
-        workouts: vec![],
-        collections: vec![],
-        failed_items: vec![],
-        measurements: vec![],
+        ..Default::default()
     })
 }
 

apps/backend/src/importer/media_tracker.rs

@@ -161,7 +161,7 @@ pub async fn import(input: DeployMediaTrackerImportInput) -> Result<ImportResult
         .unwrap();
     let mut lists: Vec<ListResponse> = rsp.body_json().await.unwrap();
 
-    let all_collections = lists
+    let collections = lists
         .iter()
         .map(|l| CreateOrUpdateCollectionInput {
             name: l.name.clone(),
@@ -368,9 +368,7 @@ pub async fn import(input: DeployMediaTrackerImportInput) -> Result<ImportResult
     Ok(ImportResult {
         media: final_data,
         failed_items,
-        collections: all_collections,
-        people: vec![],
-        workouts: vec![],
-        measurements: vec![],
+        collections,
+        ..Default::default()
     })
 }

apps/backend/src/importer/mod.rs

@@ -25,8 +25,9 @@ use crate::{
         fitness::UserWorkoutInput,
         media::{
             CommitPersonInput, CreateOrUpdateCollectionInput, ImportOrExportItemIdentifier,
-            ImportOrExportItemRating, ImportOrExportMediaItem, ImportOrExportPersonItem,
-            PartialMetadataWithoutId, PostReviewInput, ProgressUpdateInput,
+            ImportOrExportItemRating, ImportOrExportMediaGroupItem, ImportOrExportMediaItem,
+            ImportOrExportPersonItem, PartialMetadataWithoutId, PostReviewInput,
+            ProgressUpdateInput,
         },
         BackgroundJob, ChangeCollectionToEntityInput, IdObject,
     },
@@ -159,10 +160,11 @@ pub struct ImportDetails {
     pub total: usize,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Default)]
 pub struct ImportResult {
     collections: Vec<CreateOrUpdateCollectionInput>,
     media: Vec<ImportOrExportMediaItem>,
+    media_groups: Vec<ImportOrExportMediaGroupItem>,
     failed_items: Vec<ImportFailedItem>,
     people: Vec<ImportOrExportPersonItem>,
     workouts: Vec<UserWorkoutInput>,
@@ -286,6 +288,7 @@ impl ImporterService {
             }
             ImportSource::PeopleJson => self.import_people(user_id, input).await?,
             ImportSource::MeasurementsJson => self.import_measurements(user_id, input).await?,
+            ImportSource::MediaGroupJson => self.import_media_groups(user_id, input).await?,
             _ => self.import_media(user_id, input).await?,
         };
         self.media_service
@@ -328,7 +331,7 @@ impl ImporterService {
                 .await?;
             for review in item.reviews.iter() {
                 if let Some(input) =
-                    convert_review_into_input(review, &preferences, None, Some(person.id))
+                    convert_review_into_input(review, &preferences, None, Some(person.id), None)
                 {
                     if let Err(e) = self.media_service.post_review(user_id, input).await {
                         import.failed_items.push(ImportFailedItem {
@@ -434,6 +437,119 @@ impl ImporterService {
         Ok(())
     }
 
+    #[instrument(skip(self, input))]
+    async fn import_media_groups(
+        &self,
+        user_id: i32,
+        input: Box<DeployImportJobInput>,
+    ) -> Result<()> {
+        let db_import_job = self.start_import_job(user_id, input.source).await?;
+        let mut import = match input.source {
+            ImportSource::MediaGroupJson => json::media_groups_import(input.json.unwrap())
+                .await
+                .unwrap(),
+            _ => unreachable!(),
+        };
+        let preferences =
+            partial_user_by_id::<UserWithOnlyPreferences>(&self.media_service.db, user_id)
+                .await?
+                .preferences;
+        import.media = import
+            .media
+            .into_iter()
+            .sorted_unstable_by_key(|m| {
+                m.seen_history.len() + m.reviews.len() + m.collections.len()
+            })
+            .rev()
+            .collect_vec();
+        for col_details in import.collections.into_iter() {
+            self.media_service
+                .create_or_update_collection(user_id, col_details)
+                .await?;
+        }
+        for (idx, item) in import.media_groups.iter().enumerate() {
+            tracing::debug!(
+                "Importing media group with identifier = {iden}",
+                iden = &item.title
+            );
+            let rev_length = item.reviews.len();
+            let data = self
+                .media_service
+                .commit_metadata_group_internal(&item.identifier, item.lot, item.source)
+                .await;
+            let metadata_id = match data {
+                Ok(r) => r.0,
+                Err(e) => {
+                    tracing::error!("{e:?}");
+                    import.failed_items.push(ImportFailedItem {
+                        lot: Some(item.lot),
+                        step: ImportFailStep::MediaDetailsFromProvider,
+                        identifier: item.title.to_owned(),
+                        error: Some(e.message),
+                    });
+                    continue;
+                }
+            };
+            for review in item.reviews.iter() {
+                if let Some(input) =
+                    convert_review_into_input(review, &preferences, None, None, Some(metadata_id))
+                {
+                    if let Err(e) = self.media_service.post_review(user_id, input).await {
+                        import.failed_items.push(ImportFailedItem {
+                            lot: Some(item.lot),
+                            step: ImportFailStep::ReviewConversion,
+                            identifier: item.title.to_owned(),
+                            error: Some(e.message),
+                        });
+                    };
+                }
+            }
+            for col in item.collections.iter() {
+                self.media_service
+                    .create_or_update_collection(
+                        user_id,
+                        CreateOrUpdateCollectionInput {
+                            name: col.to_string(),
+                            ..Default::default()
+                        },
+                    )
+                    .await?;
+                self.media_service
+                    .add_entity_to_collection(
+                        user_id,
+                        ChangeCollectionToEntityInput {
+                            collection_name: col.to_string(),
+                            metadata_id: Some(metadata_id),
+                            ..Default::default()
+                        },
+                    )
+                    .await
+                    .ok();
+            }
+            tracing::debug!(
+                "Imported item: {idx}/{total}, lot: {lot}, review count: {rev}, collection count: {col}",
+                idx = idx + 1,
+                total = import.media.len(),
+                lot = item.lot,
+                rev = rev_length,
+                col = item.collections.len(),
+            );
+        }
+        tracing::debug!(
+            "Imported {total} media group items from {source}",
+            total = import.media.len(),
+            source = db_import_job.source
+        );
+        let details = ImportResultResponse {
+            import: ImportDetails {
+                total: import.media.len(),
+            },
+            failed_items: import.failed_items,
+        };
+        self.finish_import_job(db_import_job, details).await?;
+        Ok(())
+    }
+
     #[instrument(skip(self, input))]
     async fn import_media(&self, user_id: i32, input: Box<DeployImportJobInput>) -> Result<()> {
         let db_import_job = self.start_import_job(user_id, input.source).await?;
@@ -555,7 +671,7 @@ impl ImporterService {
             }
             for review in item.reviews.iter() {
                 if let Some(input) =
-                    convert_review_into_input(review, &preferences, Some(metadata.id), None)
+                    convert_review_into_input(review, &preferences, Some(metadata.id), None, None)
                 {
                     if let Err(e) = self.media_service.post_review(user_id, input).await {
                         import.failed_items.push(ImportFailedItem {
@@ -648,6 +764,7 @@ fn convert_review_into_input(
     preferences: &UserPreferences,
     metadata_id: Option<i32>,
     person_id: Option<i32>,
+    metadata_group_id: Option<i32>,
 ) -> Option<PostReviewInput> {
     if review.review.is_none() && review.rating.is_none() {
         tracing::debug!("Skipping review since it has no content");
@@ -668,6 +785,7 @@ fn convert_review_into_input(
         date: date.flatten(),
         metadata_id,
         person_id,
+        metadata_group_id,
         show_season_number: review.show_season_number,
         show_episode_number: review.show_episode_number,
         podcast_episode_number: review.podcast_episode_number,

apps/backend/src/importer/movary.rs

@@ -180,9 +180,6 @@ pub async fn import(input: DeployMovaryImportInput) -> Result<ImportResult> {
     Ok(ImportResult {
         media,
         failed_items,
-        people: vec![],
-        workouts: vec![],
-        collections: vec![],
-        measurements: vec![],
+        ..Default::default()
     })
 }

apps/backend/src/importer/story_graph.rs

@@ -157,9 +157,6 @@ pub async fn import(
     Ok(ImportResult {
         media,
         failed_items,
-        people: vec![],
-        workouts: vec![],
-        collections: vec![],
-        measurements: vec![],
+        ..Default::default()
     })
 }

apps/backend/src/importer/strong_app.rs

@@ -140,10 +140,6 @@ pub async fn import(
     }
     Ok(ImportResult {
         workouts,
-        media: vec![],
-        people: vec![],
-        collections: vec![],
-        failed_items: vec![],
-        measurements: vec![],
+        ..Default::default()
     })
 }