allow short-term model caching for mget requests

inaturalist · Apr 22, 2021 · 726b298 · 726b298
1 parent 411a700
commit 726b298
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 26 deletions.
diff --git a/lib/controllers/v1/taxa_controller.js b/lib/controllers/v1/taxa_controller.js
@@ -677,7 +677,8 @@ TaxaController.searchQuery = async ( req, options = { } ) => {
   if ( fetchingOnlyIDs ) {
     // use `mget` instead of `search` if looking up taxa with only IDs and no other filters
     data = await ESModel.mgetResults( fetchingOnlyIDs, "taxa", {
-      source: { _source: req._source || defaultSource }
+      source: { _source: req._source || defaultSource },
+      caching: options.caching
     } );
   } else {
     data = await esClient.connection.search( {
@@ -745,7 +746,8 @@ TaxaController.assignAncestors = async ( req, taxa, options ) => {
   const ancestorOpts = {
     filters: [{ terms: { id: ancestorIDs } }],
     per_page: ancestorIDs.length,
-    localeOpts: options.localeOpts
+    localeOpts: options.localeOpts,
+    caching: true
   };
   const newReq = Object.assign( { }, req );
   newReq.query = newReq.query || { };
@@ -770,7 +772,8 @@ TaxaController.assignChildren = async ( req, taxa, options ) => {
       { terms: { parent_id: ids } },
       { term: { is_active: true } }
     ],
-    per_page: 10000
+    per_page: 10000,
+    caching: true
   };
   const r = await TaxaController.searchQuery( req, childrenOpts );
   const childrenByID = { };

diff --git a/lib/inaturalist_api.js b/lib/inaturalist_api.js
@@ -871,7 +871,7 @@ InaturalistAPI.lookupInstance = ( req, paramKey, method, callback ) => {
     if ( ids.length !== 1 ) { return void callback( ); }
     if ( ids[0] === "any" ) { return void callback( ); }
     // lookup the instance by ID
-    method( ids[0] ).then( obj => {
+    method( ids[0], { caching: true } ).then( obj => {
       if ( !obj ) {
         return void callback( {
           error: `Unknown ${paramKey} ${req.query[paramKey]}`,

diff --git a/lib/models/es_model.js b/lib/models/es_model.js
@@ -1,5 +1,6 @@
 const _ = require( "lodash" );
 const PromisePool = require( "es6-promise-pool" );
+const md5 = require( "md5" );
 const RedisCacheClient = require( "../redis_cache_client" );
 const esClient = require( "../es_client" );
 const util = require( "../util" );
@@ -8,7 +9,7 @@ const User = require( "./user" );
 
 const ESModel = class ESModel {
   static async fetchResultsHashByIDs( ids, model, options ) {
-    // TODO: werid return value
+    // TODO: weird return value
     if ( _.isEmpty( ids ) ) { return { }; }
     const optionsFilters = _.cloneDeep( options.filters || [] );
     let should;
@@ -27,7 +28,7 @@ const ESModel = class ESModel {
     const fetchingOnlyIDs = util.filtersOnlyOnID( optionsFilters );
     if ( fetchingOnlyIDs && _.isEmpty( should ) ) {
       const source = util.sourceParams( options.source );
-      const getResponse = await ESModel.mget( fetchingOnlyIDs, model.tableName, { source } );
+      const getResponse = await ESModel.mget( fetchingOnlyIDs, model.tableName, Object.assign( { }, options, { source } ) );
       const resultsHash = { };
       _.each( getResponse.docs, h => {
         if ( h._source ) {
@@ -213,16 +214,48 @@ const ESModel = class ESModel {
     } );
   }
 
+  static async mgetCache( ids, index, cacheKeyPrefix ) {
+    const promises = _.map( ids, id => RedisCacheClient.getCompressed(
+      `${cacheKeyPrefix}-${id}`, { json: true }
+    ) );
+    const cachedModels = _.compact( await Promise.all( promises ) );
+    return _.keyBy( cachedModels, model => Number( model.id ) );
+  }
+
   static async mget( ids, index, options = { } ) {
-    let mgetParams = {
-      preference: global.config.elasticsearch.preference,
-      index: `${process.env.NODE_ENV || global.config.environment}_${index}`,
-      body: { ids }
-    };
-    if ( !_.isEmpty( options.source ) ) {
-      mgetParams = Object.assign( mgetParams, options.source );
+    let cachedModels = { };
+    let fetchedModels = { };
+    let cacheKeyPrefix = `mgetCache-${index}`;
+    if ( options.source ) {
+      cacheKeyPrefix += `-${md5( JSON.stringify( options.source ) )}`;
+    }
+    if ( options.caching ) {
+      cachedModels = await ESModel.mgetCache( ids, index, cacheKeyPrefix );
     }
-    return esClient.connection.mget( mgetParams );
+    const searchIDs = _.difference( _.map( ids, Number ), _.map( _.keys( cachedModels ), Number ) );
+    if ( !_.isEmpty( searchIDs ) ) {
+      let mgetParams = {
+        preference: global.config.elasticsearch.preference,
+        index: `${process.env.NODE_ENV || global.config.environment}_${index}`,
+        body: { ids }
+      };
+      if ( !_.isEmpty( options.source ) ) {
+        mgetParams = Object.assign( mgetParams, options.source );
+      }
+      const mgetResponse = await esClient.connection.mget( mgetParams );
+      fetchedModels = _.keyBy( _.map( _.filter( mgetResponse.docs, "_source" ), "_source" ), "id" );
+      if ( options.caching ) {
+        _.each( fetchedModels, ( model, id ) => {
+          RedisCacheClient.setCompressed(
+            `${cacheKeyPrefix}-${id}`, JSON.stringify( model ), 60
+          );
+        } );
+      }
+    }
+    const combinedDocs = _.compact( _.map( ids, id => cachedModels[id] || fetchedModels[id] ) );
+    return {
+      docs: _.map( combinedDocs, doc => ( { _source: doc } ) )
+    };
   }
 
   static async mgetResults( ids, index, options = { } ) {

diff --git a/lib/models/observation.js b/lib/models/observation.js
@@ -162,7 +162,8 @@ const Observation = class Observation extends Model {
         idFields: {
           controlled_value_id: "controlled_value",
           controlled_attribute_id: "controlled_attribute"
-        }
+        },
+        caching: true
       }
     );
   }
@@ -171,7 +172,10 @@ const Observation = class Observation extends Model {
     return ESModel.fetchBelongsTo(
       _.flattenDeep( _.map( obs, "ofvs" ) ),
       ObservationField,
-      { foreignKey: "field_id" }
+      {
+        foreignKey: "field_id",
+        caching: true
+      }
     );
   }
 
@@ -193,7 +197,8 @@ const Observation = class Observation extends Model {
       foreignKey: "community_taxon_id",
       attrName: "community_taxon",
       modifier: t => t.prepareForResponse( localeOpts ),
-      source: Taxon.esReturnFields
+      source: Taxon.esReturnFields,
+      caching: true
     };
     await ESModel.fetchBelongsTo( withProjects, Project, { source: Project.returnFields } );
     await Promise.all( [
@@ -222,7 +227,8 @@ const Observation = class Observation extends Model {
         taxon_id: "taxon",
         previous_observation_taxon_id: "previous_observation_taxon"
       },
-      source: { excludes: ["photos", "taxon_photos"] }
+      source: { excludes: ["photos", "taxon_photos"] },
+      caching: true
     };
     await ObservationPreload.identifications( obs );
     await ObservationPreload.projectObservations( obs );
@@ -261,7 +267,7 @@ const Observation = class Observation extends Model {
     const taxa = _.compact( _.map( ids, "taxon" ) );
     await Promise.all( [
       TaxaController.assignAncestors( { }, taxa, { localeOpts, ancestors: true } ),
-      ESModel.fetchBelongsTo( withUsers, User, { } ),
+      ESModel.fetchBelongsTo( withUsers, User, { caching: true } ),
       ObservationPreload.observationSounds( obs ),
       ObservationPreload.userPreferences( obs ),
       ObservationPreload.observationPhotos( obs )

diff --git a/lib/models/observation_preload.js b/lib/models/observation_preload.js
@@ -325,7 +325,9 @@ const ObservationPreload = class ObservationPreload {
     } );
     const identifications = _.filter( _.flatten( _.map( obs, "identifications" ) ), _.identity );
     await ESModel.fetchBelongsTo( identifications, Identification, {
-      foreignKey: "id", source: { excludes: ["observation", "taxon", "current_taxon"] }, forObs: true
+      foreignKey: "id",
+      source: { excludes: ["observation", "taxon", "current_taxon"] },
+      forObs: true
     } );
     _.each( obs, o => {
       o.identifications = _.filter( _.map( o.identifications, "identification" ), _.identity );

diff --git a/lib/models/place.js b/lib/models/place.js
@@ -5,12 +5,13 @@ const Model = require( "./model" );
 const ESModel = require( "./es_model" );
 
 const Place = class Place extends Model {
-  static async findByID( id ) {
+  static async findByID( id, options = { } ) {
     if ( !Number( id ) ) {
       throw new Error( "invalid place_id" );
     }
     const getResponse = await ESModel.mget( [id], "places", {
-      source: { _source: ["id", "name", "display_name", "place_type", "ancestor_place_ids"] }
+      source: { _source: ["id", "name", "display_name", "place_type", "ancestor_place_ids"] },
+      caching: options.caching
     } );
     return _.isEmpty( getResponse ) || _.isEmpty( getResponse.docs ) || !getResponse.docs[0]._source
       ? null : getResponse.docs[0]._source;

diff --git a/lib/models/taxon.js b/lib/models/taxon.js
@@ -130,7 +130,10 @@ const Taxon = class Taxon extends Model {
     } );
     const status = localStatus || ancestorStatus || globalStatus || null;
     if ( status && status.user_id && !status.user ) {
-      ESModel.fetchBelongsTo( [status], User, { source: { includes: ["id", "login"] } } );
+      ESModel.fetchBelongsTo( [status], User, {
+        source: { includes: ["id", "login"] },
+        caching: true
+      } );
     }
     return status;
   }
@@ -199,12 +202,13 @@ const Taxon = class Taxon extends Model {
     delete this.ancestors;
   }
 
-  static async findByID( id ) {
+  static async findByID( id, options = { } ) {
     if ( !Number( id ) ) {
       throw new Error( "invalid taxon_id" );
     }
     const getResponse = await ESModel.mget( [id], "taxa", {
-      source: { _source: _.union( Taxon.esReturnFields, ["min_species_ancestry"] ) }
+      source: { _source: _.union( Taxon.esReturnFields, ["min_species_ancestry"] ) },
+      caching: options.caching
     } );
     return _.isEmpty( getResponse ) || _.isEmpty( getResponse.docs ) || !getResponse.docs[0]._source
       ? null : getResponse.docs[0]._source;
@@ -432,7 +436,8 @@ const Taxon = class Taxon extends Model {
     const taxonPhotos = _.flatten( _.map( taxa, t => t.taxon_photos ) );
     const taxonOpts = {
       modifier: prepareTaxon,
-      source: _.without( Taxon.esReturnFields, "default_photo" )
+      source: _.without( Taxon.esReturnFields, "default_photo" ),
+      caching: true
     };
     await ESModel.fetchBelongsTo( taxonPhotos, Taxon, taxonOpts );
   }