feat: map AniDB IDs from Hama agent to tvdb/tmdb/imdb IDs (#538)

* feat: map AniDB IDs from Hama agent to tvdb/tmdb/imdb IDs

re #453

* refactor: removes sync job for AnimeList, load mapping on demand

* refactor: addressing review comments, using typescript types for xml parsing

* refactor: make sure sync job does not update create same tvshow/movie twice

Hama agent can have same tvdbid it for different library items - for example
when user stores different seasons for same tv show separately. This change
adds "AsyncLock" that guarantees code in callback runs for same id fully,
before running same callback next time.

* refactor: do not use season 0 tvdbid for tvshow from mapping file

* refactor: support multiple imdb mappings for same anidb entry

* refactor: add debug log for missing tvdb entries in tmdb lookups from anidb/hama agent

.gitignore

@@ -39,6 +39,9 @@ config/settings.json
 config/logs/*.log*
 config/logs/*.json
 
+# anidb mapping file
+config/anime-list.xml
+
 # dist files
 dist
 

server/api/animelist.ts

@@ -0,0 +1,223 @@
+import axios from 'axios';
+import xml2js from 'xml2js';
+import fs, { promises as fsp } from 'fs';
+import path from 'path';
+import logger from '../logger';
+
+const UPDATE_INTERVAL_MSEC = 24 * 3600 * 1000; // how often to download new mapping in milliseconds
+// originally at https://raw.githubusercontent.com/ScudLee/anime-lists/master/anime-list.xml
+const MAPPING_URL =
+  'https://raw.githubusercontent.com/Anime-Lists/anime-lists/master/anime-list.xml';
+const LOCAL_PATH = path.join(__dirname, '../../config/anime-list.xml');
+
+const mappingRegexp = new RegExp(/;[0-9]+-([0-9]+)/g);
+
+// Anime-List xml files are community maintained mappings that Hama agent uses to map AniDB IDs to tvdb/tmdb IDs
+// https://github.com/Anime-Lists/anime-lists/
+
+interface AnimeMapping {
+  $: {
+    anidbseason: string;
+    tvdbseason: string;
+  };
+  _: string;
+}
+
+interface Anime {
+  $: {
+    anidbid: number;
+    tvdbid?: string;
+    defaulttvdbseason?: string;
+    tmdbid?: number;
+    imdbid?: string;
+  };
+  'mapping-list'?: {
+    mapping: AnimeMapping[];
+  }[];
+}
+
+interface AnimeList {
+  'anime-list': {
+    anime: Anime[];
+  };
+}
+
+export interface AnidbItem {
+  tvdbId?: number;
+  tmdbId?: number;
+  imdbId?: string;
+}
+
+class AnimeListMapping {
+  private syncing = false;
+
+  private mapping: { [anidbId: number]: AnidbItem } = {};
+
+  // mapping file modification date when it was loaded
+  private mappingModified: Date | null = null;
+
+  // each episode in season 0 from TVDB can map to movie
+  private specials: { [tvdbId: number]: { [episode: number]: AnidbItem } } = {};
+
+  public isLoaded = () => Object.keys(this.mapping).length !== 0;
+
+  private loadFromFile = async () => {
+    logger.info('Loading mapping file', { label: 'Anime-List Sync' });
+    try {
+      const mappingStat = await fsp.stat(LOCAL_PATH);
+      const file = await fsp.readFile(LOCAL_PATH);
+      const xml = (await xml2js.parseStringPromise(file)) as AnimeList;
+
+      this.mapping = {};
+      this.specials = {};
+      for (const anime of xml['anime-list'].anime) {
+        // tvdbId can be nonnumber, like 'movie' string
+        let tvdbId: number | undefined;
+        if (anime.$.tvdbid && !isNaN(Number(anime.$.tvdbid))) {
+          tvdbId = Number(anime.$.tvdbid);
+        } else {
+          tvdbId = undefined;
+        }
+
+        let imdbIds: (string | undefined)[];
+        if (anime.$.imdbid) {
+          // if there are multiple imdb entries, then they map to different movies
+          imdbIds = anime.$.imdbid.split(',');
+        } else {
+          // in case there is no imdbid, that's ok as there will be tmdbid
+          imdbIds = [undefined];
+        }
+
+        const tmdbId = anime.$.tmdbid ? Number(anime.$.tmdbid) : undefined;
+        const anidbId = Number(anime.$.anidbid);
+        this.mapping[anidbId] = {
+          // for season 0 ignore tvdbid, because this must be movie/OVA
+          tvdbId: anime.$.defaulttvdbseason === '0' ? undefined : tvdbId,
+          tmdbId: tmdbId,
+          imdbId: imdbIds[0], // this is used for one AniDB -> one imdb movie mapping
+        };
+
+        if (tvdbId) {
+          const mappingList = anime['mapping-list'];
+          if (mappingList && mappingList.length != 0) {
+            let imdbIndex = 0;
+            for (const mapping of mappingList[0].mapping) {
+              const text = mapping._;
+              if (text && mapping.$.tvdbseason === '0') {
+                let matches;
+                while ((matches = mappingRegexp.exec(text)) !== null) {
+                  const episode = Number(matches[1]);
+                  if (!this.specials[tvdbId]) {
+                    this.specials[tvdbId] = {};
+                  }
+                  // map next available imdbid to episode in s0
+                  const imdbId =
+                    imdbIndex > imdbIds.length ? undefined : imdbIds[imdbIndex];
+                  if (tmdbId || imdbId) {
+                    this.specials[tvdbId][episode] = {
+                      tmdbId: tmdbId,
+                      imdbId: imdbId,
+                    };
+                    imdbIndex++;
+                  }
+                }
+              }
+            }
+          } else {
+            // some movies do not have mapping-list, so map episode 1,2,3,..to movies
+            // movies must have imdbid or tmdbid
+            const hasImdb = imdbIds.length > 1 || imdbIds[0] !== undefined;
+            if ((hasImdb || tmdbId) && anime.$.defaulttvdbseason === '0') {
+              if (!this.specials[tvdbId]) {
+                this.specials[tvdbId] = {};
+              }
+              // map each imdbid to episode in s0, episode index starts with 1
+              for (let idx = 0; idx < imdbIds.length; idx++) {
+                this.specials[tvdbId][idx + 1] = {
+                  tmdbId: tmdbId,
+                  imdbId: imdbIds[idx],
+                };
+              }
+            }
+          }
+        }
+      }
+      this.mappingModified = mappingStat.mtime;
+      logger.info(
+        `Loaded ${
+          Object.keys(this.mapping).length
+        } AniDB items from mapping file`,
+        { label: 'Anime-List Sync' }
+      );
+    } catch (e) {
+      throw new Error(`Failed to load Anime-List mappings: ${e.message}`);
+    }
+  };
+
+  private downloadFile = async () => {
+    logger.info('Downloading latest mapping file', {
+      label: 'Anime-List Sync',
+    });
+    try {
+      const response = await axios.get(MAPPING_URL, {
+        responseType: 'stream',
+      });
+      await new Promise<void>((resolve) => {
+        const writer = fs.createWriteStream(LOCAL_PATH);
+        writer.on('finish', resolve);
+        response.data.pipe(writer);
+      });
+    } catch (e) {
+      throw new Error(`Failed to download Anime-List mapping: ${e.message}`);
+    }
+  };
+
+  public sync = async () => {
+    // make sure only one sync runs at a time
+    if (this.syncing) {
+      return;
+    }
+
+    this.syncing = true;
+    try {
+      // check if local file is not "expired" yet
+      if (fs.existsSync(LOCAL_PATH)) {
+        const now = new Date();
+        const stat = await fsp.stat(LOCAL_PATH);
+        if (now.getTime() - stat.mtime.getTime() < UPDATE_INTERVAL_MSEC) {
+          if (!this.isLoaded()) {
+            // no need to download, but make sure file is loaded
+            await this.loadFromFile();
+          } else if (
+            this.mappingModified &&
+            stat.mtime.getTime() > this.mappingModified.getTime()
+          ) {
+            // if file has been modified externally since last load, reload it
+            await this.loadFromFile();
+          }
+          return;
+        }
+      }
+      await this.downloadFile();
+      await this.loadFromFile();
+    } finally {
+      this.syncing = false;
+    }
+  };
+
+  public getFromAnidbId = (anidbId: number): AnidbItem | undefined => {
+    return this.mapping[anidbId];
+  };
+
+  public getSpecialEpisode = (
+    tvdbId: number,
+    episode: number
+  ): AnidbItem | undefined => {
+    const episodes = this.specials[tvdbId];
+    return episodes ? episodes[episode] : undefined;
+  };
+}
+
+const animeList = new AnimeListMapping();
+
+export default animeList;

server/api/plexapi.ts

@@ -123,6 +123,14 @@ class PlexAPI {
     return response.MediaContainer.Metadata[0];
   }
 
+  public async getChildrenMetadata(key: string): Promise<PlexMetadata[]> {
+    const response = await this.plexClient.query<PlexMetadataResponse>(
+      `/library/metadata/${key}/children`
+    );
+
+    return response.MediaContainer.Metadata;
+  }
+
   public async getRecentlyAdded(id: string): Promise<PlexLibraryItem[]> {
     const response = await this.plexClient.query<PlexLibraryResponse>(
       `/library/sections/${id}/recentlyAdded`