feat: add support for Wikidata as a source

package.json

@@ -41,7 +41,8 @@
     "p-map": "^5.5.0",
     "p-memoize": "^7.1.1",
     "p-throttle": "^5.0.0",
-    "random": "^4.0.3"
+    "random": "^4.0.3",
+    "wikibase-sdk": "^9.2.3"
   },
   "devDependencies": {
     "@trivago/prettier-plugin-sort-imports": "^4.0.0",

src/get-wikidata-entity.ts

@@ -0,0 +1,16 @@
+import { getWikidataEntities } from './lib/wikidata'
+
+/**
+ * Quick utility to fetch an individual Wikidata movie entity.
+ */
+async function main() {
+  // const entityId = 'Q849162' // event horizon
+  // const entityId = 'Q44578' // the titanic
+  const entityId = 'Q182218' // the avengers
+
+  const [movie] = await getWikidataEntities(entityId)
+
+  console.log(JSON.stringify(movie, null, 2))
+}
+
+main()

src/lib/config.ts

@@ -14,6 +14,7 @@ export const imdbRatingsPath = `${dataDir}/title.ratings.tsv`
 // local caches
 export const imdbMoviesPath = `${outDir}/imdb-movies.json`
 export const rtMoviesPath = `${outDir}/rt-movies.json`
+export const wikidataMoviesPath = `${outDir}/wikidata-movies.json`
 export const flickMetrixMoviesPath = `${outDir}/flick-metrix-movies.json`
 export const omdbMoviesPath = `${outDir}/omdb-movies.json`
 

src/lib/flick-metrix.ts

@@ -148,17 +148,8 @@ export function populateMovieWithFlickMetrixInfo(
 
   // for these fields, we want to prioritize the flick metrix values
   const fieldOverrides: Partial<Record<types.MovieField, () => any>> = {
-    director: () => flickMetrixMovie.Director || null,
-    production: () => flickMetrixMovie.Production || null,
-    awardsSummary: () => flickMetrixMovie.Awards || null,
-    letterboxdScore: () => flickMetrixMovie.LetterboxdScore ?? null,
-    letterboxdVotes: () => flickMetrixMovie.letterboxdVotes ?? null,
     flickMetrixId: () => flickMetrixMovie.ID || null,
-    flickMetrixScore: () => flickMetrixMovie.ComboScore ?? null,
-    cast: () =>
-      flickMetrixMovie.Cast?.split(',')
-        .map((name) => name.trim())
-        .filter(Boolean) ?? []
+    flickMetrixScore: () => flickMetrixMovie.ComboScore ?? null
   }
 
   for (const [field, valueFn] of Object.entries(fieldOverrides)) {
@@ -171,20 +162,30 @@ export function populateMovieWithFlickMetrixInfo(
 
   // for these fields, we want to prioritize values from other sources
   const fieldOptionals: Partial<Record<types.MovieField, () => any>> = {
+    plot: () => flickMetrixMovie.Plot ?? null,
+    director: () => flickMetrixMovie.Director || null,
+    production: () => flickMetrixMovie.Production || null,
+    awardsSummary: () => flickMetrixMovie.Awards || null,
+    letterboxdScore: () => flickMetrixMovie.LetterboxdScore ?? null,
+    letterboxdVotes: () => flickMetrixMovie.letterboxdVotes ?? null,
     rtCriticRating: () => flickMetrixMovie.CriticRating ?? null,
     rtCriticVotes: () => flickMetrixMovie.CriticReviews ?? null,
     rtAudienceRating: () => flickMetrixMovie.AudienceRating ?? null,
     rtAudienceVotes: () => flickMetrixMovie.AudienceReviews ?? null,
-    rtUrl: () => flickMetrixMovie.RTUrl.replace(/\/$/g, '').trim() || null,
-    plot: () => flickMetrixMovie.Plot ?? null,
+    rtUrl: () => flickMetrixMovie.RTUrl?.replace(/\/$/g, '').trim() || null,
     imdbRating: () => flickMetrixMovie.imdbRating ?? null,
-    imdbVotes: () => flickMetrixMovie.imdbVotes ?? null
+    imdbVotes: () => flickMetrixMovie.imdbVotes ?? null,
+    cast: () =>
+      flickMetrixMovie.Cast?.split(',')
+        .map((name) => name.trim())
+        .filter(Boolean) ?? []
   }
 
-  for (const [field, valueFn] of Object.entries(fieldOverrides)) {
+  for (const [field, valueFn] of Object.entries(fieldOptionals)) {
+    if (movie[field]) continue
     const value = valueFn()
 
-    if (!movie[field] && (value || value === 0)) {
+    if (value || value === 0) {
       ;(movie as any)[field] = value
     }
   }

src/lib/rt.ts

@@ -262,9 +262,10 @@ export function populateMovieWithRTInfo(
   const fieldOptionals: types.MovieField[] = ['title', 'mpaaRating', 'plot']
 
   for (const field of fieldOptionals) {
+    if (movie[field]) continue
     const value = rtMovie[field]
 
-    if (!movie[field] && (value || value === 0)) {
+    if (value || value === 0) {
       ;(movie as any)[field] = value
     }
   }

src/lib/utils.ts

@@ -76,6 +76,8 @@ export function convertTMDBMovieDetailsToMovie(
     id: movieDetails.id,
     tmdbId: movieDetails.id,
     imdbId: movieDetails.imdb_id,
+
+    // external ids
     wikidataId: movieDetails.external_ids?.wikidata_id,
     facebookId: movieDetails.external_ids?.facebook_id,
     instagramId: movieDetails.external_ids?.instagram_id,