Deduplicate state changes (#812)

* feat(backend): task to perform all background tasks

* feat(frontend): add btn for new admin job

* refactor(frontend): extract common prop

* feat(database): add migrations for state changes

* build(backend): bump version

* chore: change refs of fly domain to custom

* feat(backend): add new columns to entities

* feat(backend): deduplicate state changes for people

* docs: change working about fly

* fix(backend): log the plex payload

README.md

@@ -23,24 +23,18 @@
 <p align="center">
     <a href="https://docs.ryot.io" target="_blank">Installation</a> •
     <a href="https://docs.ryot.io/configuration" target="_blank">Configuration</a> •
-    <a href="https://ryot.fly.dev" target="_blank">Demo</a>
+    <a href="https://demo.ryot.io" target="_blank">Demo</a>
 </p>
 
 <br/>
 
 Ryot (**R**oll **Y**our **O**wn **T**racker), pronounced "riot", aims to be the only self
 hosted tracker you will ever need!
 
-## IMPORTANT NOTE FOR `v4.*` USERS
-
-If you were using `v4.*` of Ryot, please read the [migration
-guide](https://docs.ryot.io/migration.html#from-v4-to-v5) for instructions.
-
 ## 💻 Demo
 
-You can use the demo account on [Fly.io](https://ryot.fly.dev). Login with the username
-`demo` and password `demo-password`. This instance is automatically deployed from the
-latest release.
+You can use the demo account on the Ryot website. Login with the username `demo` and
+password `demo-password`. This instance is automatically deployed from the latest release.
 
 **NOTE**: The data in this demo account is not recommended to be used for any production
 usage. Please create a new account for that.

apps/backend/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ryot"
-version = "5.1.2"
+version = "5.2.0"
 edition = "2021"
 repository = "https://github.com/IgnisDa/ryot"
 license = "GPL-3.0"

apps/backend/src/background.rs

@@ -74,13 +74,7 @@ pub async fn media_jobs(_information: ScheduledJob, ctx: JobContext) -> Result<(
 
 pub async fn user_jobs(_information: ScheduledJob, ctx: JobContext) -> Result<(), JobError> {
     let misc_service = ctx.data::<Arc<MiscellaneousService>>().unwrap();
-    tracing::trace!("Cleaning up user and metadata association");
-    misc_service
-        .cleanup_user_and_metadata_association()
-        .await
-        .unwrap();
-    tracing::trace!("Removing old user summaries and regenerating them");
-    misc_service.regenerate_user_summaries().await.unwrap();
+    misc_service.perform_user_jobs().await.unwrap();
     Ok(())
 }
 
@@ -147,6 +141,7 @@ pub enum ApplicationJob {
     ReviewPosted(ReviewPostedEvent),
     PerformExport(i32, Vec<ExportItem>),
     RecalculateUserSummary(i32),
+    PerformUserBackgroundTasks,
 }
 
 impl Job for ApplicationJob {
@@ -192,6 +187,9 @@ pub async fn perform_application_job(
         ApplicationJob::RecalculateCalendarEvents => {
             misc_service.recalculate_calendar_events().await.is_ok()
         }
+        ApplicationJob::PerformUserBackgroundTasks => {
+            misc_service.perform_user_jobs().await.is_ok()
+        }
         ApplicationJob::AssociateGroupWithMetadata(lot, source, identifier) => misc_service
             .commit_metadata_group(CommitMediaInput {
                 lot,

apps/backend/src/entities/metadata.rs

@@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize};
 
 use crate::models::media::{
     AnimeSpecifics, AudioBookSpecifics, BookSpecifics, MangaSpecifics, MetadataFreeCreator,
-    MetadataImage, MetadataVideo, MovieSpecifics, PodcastSpecifics, ShowSpecifics,
-    VideoGameSpecifics, VisualNovelSpecifics, WatchProvider,
+    MetadataImage, MetadataStateChanges, MetadataVideo, MovieSpecifics, PodcastSpecifics,
+    ShowSpecifics, VideoGameSpecifics, VisualNovelSpecifics, WatchProvider,
 };
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize, Default)]
@@ -48,6 +48,7 @@ pub struct Model {
     pub visual_novel_specifics: Option<VisualNovelSpecifics>,
     pub anime_specifics: Option<AnimeSpecifics>,
     pub manga_specifics: Option<MangaSpecifics>,
+    pub state_changes: Option<MetadataStateChanges>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

apps/backend/src/entities/person.rs

@@ -6,7 +6,7 @@ use database::MediaSource;
 use sea_orm::entity::prelude::*;
 use serde::{Deserialize, Serialize};
 
-use crate::models::media::{MetadataImage, PersonSourceSpecifics};
+use crate::models::media::{MetadataImage, PersonSourceSpecifics, PersonStateChanges};
 
 #[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize, SimpleObject)]
 #[graphql(name = "Person")]
@@ -33,6 +33,8 @@ pub struct Model {
     pub website: Option<String>,
     #[graphql(skip)]
     pub source_specifics: Option<PersonSourceSpecifics>,
+    #[graphql(skip)]
+    pub state_changes: Option<PersonStateChanges>,
 }
 
 #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

apps/backend/src/integrations.rs

@@ -165,6 +165,8 @@ impl IntegrationService {
             }
         }
 
+        tracing::debug!("Processing Plex payload {:#?}", payload);
+
         let payload_regex = Regex::new(r"\{.*\}").unwrap();
         let json_payload = payload_regex
             .find(payload)

apps/backend/src/miscellaneous/resolver.rs

@@ -2733,18 +2733,16 @@ impl MiscellaneousService {
         let core_sqlite_storage = &mut self.perform_core_application_job.clone();
         let sqlite_storage = &mut self.perform_application_job.clone();
         match job_name {
-            BackgroundJob::YankIntegrationsData => {
-                core_sqlite_storage
-                    .push(CoreApplicationJob::YankIntegrationsData(user_id))
-                    .await?;
-            }
-            BackgroundJob::CalculateSummary => {
-                sqlite_storage
-                    .push(ApplicationJob::RecalculateUserSummary(user_id))
-                    .await?;
+            BackgroundJob::UpdateAllMetadata
+            | BackgroundJob::UpdateAllExercises
+            | BackgroundJob::RecalculateCalendarEvents
+            | BackgroundJob::PerformUserBackgroundTasks => {
+                self.admin_account_guard(user_id).await?;
             }
+            _ => {}
+        }
+        match job_name {
             BackgroundJob::UpdateAllMetadata => {
-                self.admin_account_guard(user_id).await?;
                 let many_metadata = Metadata::find()
                     .select_only()
                     .column(metadata::Column::Id)
@@ -2758,7 +2756,6 @@ impl MiscellaneousService {
                 }
             }
             BackgroundJob::UpdateAllExercises => {
-                self.admin_account_guard(user_id).await?;
                 let service = ExerciseService::new(
                     &self.db,
                     self.config.clone(),
@@ -2768,11 +2765,25 @@ impl MiscellaneousService {
                 service.deploy_update_exercise_library_job().await?;
             }
             BackgroundJob::RecalculateCalendarEvents => {
-                self.admin_account_guard(user_id).await?;
                 sqlite_storage
                     .push(ApplicationJob::RecalculateCalendarEvents)
                     .await?;
             }
+            BackgroundJob::PerformUserBackgroundTasks => {
+                sqlite_storage
+                    .push(ApplicationJob::PerformUserBackgroundTasks)
+                    .await?;
+            }
+            BackgroundJob::YankIntegrationsData => {
+                core_sqlite_storage
+                    .push(CoreApplicationJob::YankIntegrationsData(user_id))
+                    .await?;
+            }
+            BackgroundJob::CalculateSummary => {
+                sqlite_storage
+                    .push(ApplicationJob::RecalculateUserSummary(user_id))
+                    .await?;
+            }
             BackgroundJob::EvaluateWorkouts => {
                 sqlite_storage
                     .push(ApplicationJob::ReEvaluateUserWorkouts(user_id))
@@ -7203,6 +7214,7 @@ impl MiscellaneousService {
                 })
                 .collect()
         });
+        let mut default_state_changes = person.clone().state_changes.unwrap_or_default();
         let mut to_update_person: person::ActiveModel = person.clone().into();
         to_update_person.last_updated_on = ActiveValue::Set(Utc::now());
         to_update_person.description = ActiveValue::Set(provider_person.description);
@@ -7214,7 +7226,6 @@ impl MiscellaneousService {
         to_update_person.images = ActiveValue::Set(images);
         to_update_person.is_partial = ActiveValue::Set(Some(false));
         to_update_person.name = ActiveValue::Set(provider_person.name);
-        to_update_person.update(&self.db).await.unwrap();
         for (role, media) in provider_person.related.clone() {
             let title = media.title.clone();
             let pm = self.create_partial_metadata(media).await?;
@@ -7225,22 +7236,28 @@ impl MiscellaneousService {
                 .one(&self.db)
                 .await?;
             if already_intermediate.is_none() {
+                let intermediate = metadata_to_person::ActiveModel {
+                    person_id: ActiveValue::Set(person.id),
+                    metadata_id: ActiveValue::Set(pm.id),
+                    role: ActiveValue::Set(role.clone()),
+                    ..Default::default()
+                };
+                intermediate.insert(&self.db).await.unwrap();
+            }
+            let search_for = (pm.id, role.clone());
+            if !default_state_changes.media_associated.contains(&search_for) {
                 notifications.push((
                     format!(
                         "{} has been associated with {} as {}",
                         person.name, title, role
                     ),
                     MediaStateChanged::PersonMediaAssociated,
                 ));
-                let intermediate = metadata_to_person::ActiveModel {
-                    person_id: ActiveValue::Set(person.id),
-                    metadata_id: ActiveValue::Set(pm.id),
-                    role: ActiveValue::Set(role),
-                    ..Default::default()
-                };
-                intermediate.insert(&self.db).await.unwrap();
+                default_state_changes.media_associated.insert(search_for);
             }
         }
+        to_update_person.state_changes = ActiveValue::Set(Some(default_state_changes));
+        to_update_person.update(&self.db).await.unwrap();
         Ok(notifications)
     }
 
@@ -7417,6 +7434,14 @@ GROUP BY m.id;
         }
     }
 
+    pub async fn perform_user_jobs(&self) -> Result<()> {
+        tracing::trace!("Cleaning up user and metadata association");
+        self.cleanup_user_and_metadata_association().await?;
+        tracing::trace!("Removing old user summaries and regenerating them");
+        self.regenerate_user_summaries().await?;
+        Ok(())
+    }
+
     #[cfg(debug_assertions)]
     async fn development_mutation(&self) -> Result<bool> {
         Ok(true)

apps/backend/src/models.rs

@@ -41,6 +41,7 @@ pub enum BackgroundJob {
     UpdateAllExercises,
     RecalculateCalendarEvents,
     YankIntegrationsData,
+    PerformUserBackgroundTasks,
 }
 
 #[derive(Enum, Clone, Debug, Copy, PartialEq, Eq, Serialize, Deserialize, Default, Display)]
@@ -1382,6 +1383,14 @@ pub mod media {
         #[graphql(skip_input)]
         pub force_update: Option<bool>,
     }
+
+    #[derive(Debug, Serialize, Deserialize, Clone, FromJsonQueryResult, Eq, PartialEq, Default)]
+    pub struct MetadataStateChanges {}
+
+    #[derive(Debug, Serialize, Deserialize, Clone, FromJsonQueryResult, Eq, PartialEq, Default)]
+    pub struct PersonStateChanges {
+        pub media_associated: HashSet<(i32, String)>,
+    }
 }
 
 pub mod fitness {

apps/frontend/app/routes/_dashboard.settings.miscellaneous.tsx

@@ -81,11 +81,7 @@ export default function Page() {
 											providers.
 										</Text>
 									</Box>
-									<Button
-										{...buttonProps}
-										name="jobName"
-										value={BackgroundJob.UpdateAllMetadata}
-									>
+									<Button {...btnProps} value={BackgroundJob.UpdateAllMetadata}>
 										Update metadata
 									</Button>
 								</Stack>
@@ -100,8 +96,7 @@ export default function Page() {
 										</Text>
 									</Box>
 									<Button
-										{...buttonProps}
-										name="jobName"
+										{...btnProps}
 										value={BackgroundJob.RecalculateCalendarEvents}
 									>
 										Update calendar events
@@ -117,13 +112,28 @@ export default function Page() {
 										</Text>
 									</Box>
 									<Button
-										{...buttonProps}
-										name="jobName"
+										{...btnProps}
 										value={BackgroundJob.UpdateAllExercises}
 									>
 										Update exercises
 									</Button>
 								</Stack>
+								<Stack>
+									<Box>
+										<Title order={4}>Jobs for all users</Title>
+										<Text>
+											Update the user summaries. Also recalculate the media
+											associations for all users. The more users you have, the
+											longer this will take.
+										</Text>
+									</Box>
+									<Button
+										{...btnProps}
+										value={BackgroundJob.PerformUserBackgroundTasks}
+									>
+										Perform user tasks
+									</Button>
+								</Stack>
 							</>
 						) : null}
 						<Stack>
@@ -135,11 +145,7 @@ export default function Page() {
 									preconditions have changed. This may take some time.
 								</Text>
 							</Box>
-							<Button
-								{...buttonProps}
-								name="jobName"
-								value={BackgroundJob.CalculateSummary}
-							>
+							<Button {...btnProps} value={BackgroundJob.CalculateSummary}>
 								Clean and regenerate
 							</Button>
 						</Stack>
@@ -152,11 +158,7 @@ export default function Page() {
 									deleted.
 								</Text>
 							</Box>
-							<Button
-								{...buttonProps}
-								name="jobName"
-								value={BackgroundJob.EvaluateWorkouts}
-							>
+							<Button {...btnProps} value={BackgroundJob.EvaluateWorkouts}>
 								Re-evaluate workouts
 							</Button>
 						</Stack>
@@ -169,11 +171,7 @@ export default function Page() {
 									longer this will take.
 								</Text>
 							</Box>
-							<Button
-								{...buttonProps}
-								name="jobName"
-								value={BackgroundJob.YankIntegrationsData}
-							>
+							<Button {...btnProps} value={BackgroundJob.YankIntegrationsData}>
 								Synchronize
 							</Button>
 						</Stack>
@@ -184,4 +182,8 @@ export default function Page() {
 	);
 }
 
-const buttonProps = { variant: "light", type: "submit" as const };
+const btnProps = {
+	variant: "light",
+	type: "submit" as const,
+	name: "jobName",
+};

docs/content/configuration.md

@@ -4,7 +4,7 @@ You can specify configuration options via environment variables or via files (lo
 `config/ryot.json`, `config/ryot.toml`, `config/ryot.yaml`). They should be present in `/home/ryot/config/ryot.<ext>`.
 
 Ryot serves the final configuration loaded at the `/backend/config` endpoint as JSON
-([example](https://ryot.fly.dev/backend/config)). This can also be treated as a [health
+([example](https://demo.ryot.io/backend/config)). This can also be treated as a [health
 endpoint](https://learn.microsoft.com/en-us/azure/architecture/patterns/health-endpoint-monitoring).
 
 !!! info

docs/content/guides/openid.md

@@ -4,7 +4,7 @@ Ryot can be configured to use OpenID Connect (OIDC) for authentication. The foll
 environment variables need to be set:
 
 ```bash
-FRONTEND_URL="https://ryot.fly.dev" # The URL of your Ryot instance
+FRONTEND_URL="https://demo.ryot.io" # The URL of your Ryot instance
 SERVER_OIDC_CLIENT_ID="********"
 SERVER_OIDC_CLIENT_SECRET="********"
 SERVER_OIDC_ISSUER_URL="https://accounts.google.com" # The URL of your OIDC provider