Improve SSP performance by caching entity lookups

app/models/change_payload.rb

@@ -42,21 +42,22 @@ class ChangePayload < ActiveRecord::Base
   })
 
   def apply!
+    cache = {}
+    entity_types = {
+      feed: Feed,
+      stop: Stop,
+      operator: Operator,
+      route: Route,
+      schedule_stop_pair: ScheduleStopPair
+    }
     (payload_as_ruby_hash[:changes] || []).each do |change|
-      if change[:feed].present?
-        Feed.apply_change(changeset: changeset, attrs: change[:feed], action: change[:action])
-      end
-      if change[:stop].present?
-        Stop.apply_change(changeset: changeset, attrs: change[:stop], action: change[:action])
-      end
-      if change[:operator].present?
-        Operator.apply_change(changeset: changeset, attrs: change[:operator], action: change[:action])
-      end
-      if change[:route].present?
-        Route.apply_change(changeset: changeset, attrs: change[:route], action: change[:action])
-      end
-      if change[:schedule_stop_pair].present?
-        ScheduleStopPair.apply_change(changeset: changeset, attrs: change[:schedule_stop_pair], action: change[:action])
+      (entity_types.keys & change.keys).each do |entity_type|
+        entity_types[entity_type].apply_change(
+          changeset: changeset,
+          attrs: change[entity_type],
+          action: change[:action],
+          cache: cache
+        )
       end
     end
   end

app/models/concerns/current_tracked_by_changeset.rb

@@ -13,11 +13,11 @@ module ClassMethods
     attr_reader :kind_of_model_tracked,
                 :virtual_attributes
 
-    def apply_change(changeset: nil, attrs: {}, action: nil)
+    def apply_change(changeset: nil, attrs: {}, action: nil, cache: {})
       existing_model = find_existing_model(attrs)
       case action
       when 'createUpdate'
-        attrs_to_apply = attrs.select { |key, value| self.changeable_attributes.include?(key) }
+        attrs_to_apply = apply_params(attrs, cache)
         if existing_model
           existing_model.update_making_history(changeset: changeset, new_attrs: attrs_to_apply)
         else
@@ -34,6 +34,11 @@ def apply_change(changeset: nil, attrs: {}, action: nil)
       end
     end
 
+    def apply_params(params, cache={})
+      # Filter changeset params
+      params.select { |key, value| self.changeable_attributes.include?(key) }
+    end
+
     def before_create_making_history(instantiated_model, changeset)
       # this is available for overriding in models
       super(instantiated_model, changeset) if defined?(super)

app/models/concerns/is_an_entity_imported_from_feeds.rb

@@ -8,14 +8,11 @@ module IsAnEntityImportedFromFeeds
   end
 
   def imported_from_feed=(imported_from_feed_params)
-    imported_from_feed_params = HashHelpers::update_keys(imported_from_feed_params, :underscore)
-    eiff_params = {}
-    feed = Feed.find_by!(onestop_id: imported_from_feed_params[:onestop_id])
-    eiff_params[:feed] = feed
-    if imported_from_feed_params[:sha1].present?
-      feed_version = feed.feed_versions.find_by!(sha1: imported_from_feed_params[:sha1])
-      eiff_params[:feed_version] = feed_version
+    params = HashHelpers::update_keys(imported_from_feed_params, :underscore)
+    feed = params[:feed] || Feed.find_by!(onestop_id: params[:onestop_id])
+    if params[:sha1].present?
+      feed_version = params[:feed_version] || feed.feed_versions.find_by!(sha1: params[:sha1])
     end
-    self.entities_imported_from_feed.find_or_initialize_by(eiff_params)
+    self.entities_imported_from_feed.find_or_initialize_by(feed: feed, feed_version: feed_version)
   end
 end

app/models/schedule_stop_pair.rb

@@ -194,6 +194,27 @@ def self.find_by_attributes(attrs = {})
       find(attrs[:id])
     end
   end
+  def self.apply_params(params, cache={})
+    params = super(params, cache)
+    {
+      origin_onestop_id: :origin,
+      destination_onestop_id: :destination,
+      route_onestop_id: :route
+    }.each do |k,v|
+      cache[params[k]] ||= OnestopId.find!(params[k])
+      params[v] = cache[params.delete(k)]
+    end
+    if params[:imported_from_feed]
+      feed_onestop_id = params[:imported_from_feed][:onestop_id]
+      feed_version_id = params[:imported_from_feed][:sha1]
+      cache[feed_onestop_id] ||= OnestopId.find!(feed_onestop_id)
+      cache[feed_version_id] ||= cache[feed_onestop_id].feed_versions.find_by!(sha1: feed_version_id)
+      params[:imported_from_feed][:feed] = cache[feed_onestop_id]
+      params[:imported_from_feed][:feed_version] = cache[feed_version_id]
+    end
+    params[:operator] = params[:route].operator
+    params
+  end
 
   # Interpolate
   def self.interpolate(ssps, method=:linear)

app/services/gtfs_graph.rb

@@ -44,8 +44,9 @@ def create_change_osr(import_level=0)
       create_change_payloads(changeset, 'route', routes.map { |e| make_change_route(e) })
     end
     log "Changeset apply"
+    t = Time.now
     changeset.apply!
-    log "  changeset apply done"
+    log "  apply done: time #{Time.now - t}"
   end
 
   def ssp_schedule_async
@@ -96,8 +97,9 @@ def ssp_perform_async(trip_ids, agency_map, route_map, stop_map)
       create_change_payloads(changeset, 'scheduleStopPair', ssps.map { |e| make_change_ssp(e) })
     end
     log "Changeset apply"
+    t = Time.now
     changeset.apply!
-    log "  changeset apply done"
+    log "  apply done: total time: #{Time.now - t}"
   end
 
   def import_log