Merge pull request #854 from transitland/import-gtfs-identifiers

EIFF: Include GTFS identifier
transitland · Dec 12, 2016 · d0fd97c · d0fd97c
2 parents 933f6bc + a846f43
commit d0fd97c
Show file tree

Hide file tree

Showing 22 changed files with 235 additions and 74 deletions.
diff --git a/app/models/changeset.rb b/app/models/changeset.rb
@@ -260,22 +260,6 @@ def cycle_issues(issues_changeset_is_resolving, new_issues_created_by_changeset,
     end
   end
 
-  def create_feed_entity_associations
-    if import?
-      eiff_batch = []
-      self.entities_created_or_updated do |entity|
-        eiff_batch << entity
-          .entities_imported_from_feed
-          .new(feed: self.imported_from_feed, feed_version: self.imported_from_feed_version)
-        if eiff_batch.size >= 1000
-          EntityImportedFromFeed.import eiff_batch
-          eiff_batch = []
-        end
-      end
-      EntityImportedFromFeed.import eiff_batch
-    end
-  end
-
   def apply!
     fail Changeset::Error.new(changeset: self, message: 'has already been applied.') if applied
     new_issues_created_by_changeset = []
@@ -292,8 +276,6 @@ def apply!
         end
         self.update(applied: true, applied_at: Time.now)
 
-        create_feed_entity_associations
-
         # Update attributes that derive from attributes between models
         # This needs to be done before quality checks. Only on import.
         unless import?

diff --git a/app/models/concerns/is_an_entity_imported_from_feeds.rb b/app/models/concerns/is_an_entity_imported_from_feeds.rb
@@ -2,8 +2,9 @@ module IsAnEntityImportedFromFeeds
   extend ActiveSupport::Concern
   included do
     has_many :entities_imported_from_feed, as: :entity
-    has_many :imported_from_feeds, through: :entities_imported_from_feed, source: :feed
-    has_many :imported_from_feed_versions, through: :entities_imported_from_feed, source: :feed_version
+    has_many :imported_from_feeds, -> { distinct }, through: :entities_imported_from_feed, source: :feed
+    has_many :imported_from_feed_versions, -> { distinct }, through: :entities_imported_from_feed, source: :feed_version
+
     scope :where_import_level, -> (import_level) {
       joins(:entities_imported_from_feed)
         .where(entities_imported_from_feed: {
@@ -37,5 +38,26 @@ module IsAnEntityImportedFromFeeds
       where(id: self.all.select(:id).pluck(:id) - self.where_imported_from_active_feed_version.select(:id).pluck(:id))
     }
 
+    attr_accessor :add_imported_from_feeds, :not_imported_from_feeds
+    def update_entity_imported_from_feeds(changeset)
+      (self.add_imported_from_feeds || []).uniq.each do |eiff|
+        feed_version = FeedVersion.find_by!(sha1: eiff[:feed_version])
+        gtfs_id = eiff[:gtfs_id]
+        self.entities_imported_from_feed.find_or_create_by!(
+          feed_id: feed_version.feed_id,
+          feed_version_id: feed_version.id,
+          gtfs_id: gtfs_id
+        )
+      end
+      (self.not_imported_from_feeds || []).uniq.each do |eiff|
+        feed_version = FeedVersion.find_by!(sha1: eiff[:feed_version])
+        gtfs_id = eiff[:gtfs_id]
+        self.entities_imported_from_feed.find_by!(
+          feed_id: feed_version.feed_id,
+          feed_version_id: feed_version.id,
+          gtfs_id: gtfs_id
+        ).delete
+      end
+    end
   end
 end
diff --git a/app/models/entity_imported_from_feed.rb b/app/models/entity_imported_from_feed.rb
@@ -9,6 +9,7 @@
 #  created_at      :datetime
 #  updated_at      :datetime
 #  feed_version_id :integer
+#  gtfs_id         :string
 #
 # Indexes
 #
@@ -22,5 +23,9 @@ class EntityImportedFromFeed < ActiveRecord::Base
   belongs_to :feed
   belongs_to :feed_version
 
-  # validates :entity_id, uniqueness: { scope: [:feed_id, :feed_version_id] }
+  validates :entity_id, uniqueness: { scope: [:entity_type, :feed_id, :feed_version_id, :gtfs_id] }
+  validates :entity,
+            :feed,
+            :feed_version,
+            presence: true
 end
diff --git a/app/models/feed.rb b/app/models/feed.rb
@@ -66,11 +66,10 @@ class Feed < BaseFeed
   has_many :issues, through: :entities_with_issues
 
   has_many :entities_imported_from_feed
-  has_many :imported_operators, through: :entities_imported_from_feed, source: :entity, source_type: 'Operator'
-  has_many :imported_stops, through: :entities_imported_from_feed, source: :entity, source_type: 'Stop'
-  has_many :imported_routes, through: :entities_imported_from_feed, source: :entity, source_type: 'Route'
-  has_many :imported_schedule_stop_pairs, through: :entities_imported_from_feed, source: :entity, source_type: 'ScheduleStopPair'
-  has_many :imported_route_stop_patterns, through: :entities_imported_from_feed, source: :entity, source_type: 'RouteStopPattern'
+  has_many :imported_operators, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'Operator'
+  has_many :imported_stops, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'Stop'
+  has_many :imported_routes, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'Route'
+  has_many :imported_route_stop_patterns, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'RouteStopPattern'
   has_many :imported_schedule_stop_pairs, class_name: 'ScheduleStopPair', dependent: :delete_all
 
   has_many :changesets_imported_from_this_feed, class_name: 'Changeset'

diff --git a/app/models/feed_version.rb b/app/models/feed_version.rb
@@ -35,10 +35,12 @@ class FeedVersion < ActiveRecord::Base
   belongs_to :feed, polymorphic: true
   has_many :feed_version_imports, -> { order 'created_at DESC' }, dependent: :destroy
   has_many :changesets_imported_from_this_feed_version, class_name: 'Changeset'
+
   has_many :entities_imported_from_feed
-  has_many :imported_operators, through: :entities_imported_from_feed, source: :entity, source_type: 'Operator'
-  has_many :imported_stops, through: :entities_imported_from_feed, source: :entity, source_type: 'Stop'
-  has_many :imported_routes, through: :entities_imported_from_feed, source: :entity, source_type: 'Route'
+  has_many :imported_operators, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'Operator'
+  has_many :imported_stops, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'Stop'
+  has_many :imported_routes, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'Route'
+  has_many :imported_route_stop_patterns, -> { distinct }, through: :entities_imported_from_feed, source: :entity, source_type: 'RouteStopPattern'
   has_many :imported_schedule_stop_pairs, class_name: 'ScheduleStopPair', dependent: :delete_all
 
   mount_uploader :file, FeedVersionUploader

diff --git a/app/models/json_schemas/imported_from_feed.json b/app/models/json_schemas/imported_from_feed.json
@@ -0,0 +1,15 @@
+{
+	"$schema": "http://json-schema.org/draft-04/schema#",
+	"title": "Transitland: Imported From Feed",
+	"type": "object",
+	"required": ["feedVersion"],
+	"additionalProperties": false,
+	"properties": {
+		"feedVersion": {
+			"type": "string"
+		},
+		"gtfsId": {
+			"type": ["string", "null"]
+		}
+	}
+}
diff --git a/app/models/json_schemas/operator.json b/app/models/json_schemas/operator.json
@@ -17,6 +17,22 @@
     "timezone": { "type": "string" },
     "tags": { "type": "object" },
     "geometry": { "type": "object" },
+    "addImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
+    "notImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
     "identifiedBy": {
       "type": "array",
       "uniqueItems": true,

diff --git a/app/models/json_schemas/route.json b/app/models/json_schemas/route.json
@@ -29,6 +29,22 @@
       "type": ["string", "integer"],
       "format": "vehicle-type"
     },
+    "addImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
+    "notImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
     "identifiedBy": {
       "type": "array",
       "uniqueItems": true,

diff --git a/app/models/json_schemas/route_stop_pattern.json b/app/models/json_schemas/route_stop_pattern.json
@@ -26,6 +26,20 @@
     "isGenerated": {
       "type": "boolean"
     },
+    "addImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
+    "notImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
     "identifiedBy": {
       "type": "array",
       "uniqueItems": true,

diff --git a/app/models/json_schemas/stop.json b/app/models/json_schemas/stop.json
@@ -18,6 +18,22 @@
       "type": "string",
       "format": "stop-onestop-id"
     },
+    "addImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
+    "notImportedFromFeeds": {
+      "type": "array",
+      "minItems": 1,
+      "uniqueItems": true,
+      "items": {
+        "$ref": "./imported_from_feed.json"
+      }
+    },
     "identifiedBy": {
       "type": ["array", "null"],
       "uniqueItems": true,

diff --git a/app/models/operator.rb b/app/models/operator.rb
@@ -70,7 +70,9 @@ def csv_row_values
       :serves,
       :does_not_serve,
       :identified_by,
-      :not_identified_by
+      :not_identified_by,
+      :add_imported_from_feeds,
+      :not_imported_from_feeds
     ],
     protected_attributes: [
       :identifiers
@@ -84,6 +86,7 @@ def csv_row_values
     ]
   })
   def after_create_making_history(changeset)
+    update_entity_imported_from_feeds(changeset)
     OperatorRouteStopRelationship.manage_multiple(
       operator: {
         serves: self.serves || [],
@@ -94,6 +97,7 @@ def after_create_making_history(changeset)
     )
   end
   def before_update_making_history(changeset)
+    update_entity_imported_from_feeds(changeset)
     OperatorRouteStopRelationship.manage_multiple(
       operator: {
         serves: self.serves || [],

diff --git a/app/models/route.rb b/app/models/route.rb
@@ -85,7 +85,9 @@ def csv_row_values
       :does_not_serve,
       :operated_by,
       :identified_by,
-      :not_identified_by
+      :not_identified_by,
+      :add_imported_from_feeds,
+      :not_imported_from_feeds
     ],
     protected_attributes: [
       :identifiers
@@ -109,6 +111,7 @@ def self.before_create_making_history(new_model, changeset)
     self.existing_before_create_making_history(new_model, changeset)
   end
   def after_create_making_history(changeset)
+    update_entity_imported_from_feeds(changeset)
     OperatorRouteStopRelationship.manage_multiple(
       route: {
         serves: self.serves || [],
@@ -119,6 +122,7 @@ def after_create_making_history(changeset)
     )
   end
   def before_update_making_history(changeset)
+    update_entity_imported_from_feeds(changeset)
     if self.operated_by.present?
       operator = Operator.find_by_onestop_id!(self.operated_by)
       self.operator = operator

diff --git a/app/models/route_stop_pattern.rb b/app/models/route_stop_pattern.rb
@@ -83,7 +83,9 @@ def correct_stop_distances_length
     virtual_attributes: [
       :identified_by,
       :not_identified_by,
-      :traversed_by
+      :traversed_by,
+      :add_imported_from_feeds,
+      :not_imported_from_feeds
     ],
     protected_attributes: [
       :identifiers
@@ -92,14 +94,24 @@ def correct_stop_distances_length
       :geometry
     ]
   })
+
   class << RouteStopPattern
     alias_method :existing_before_create_making_history, :before_create_making_history
   end
+
+  def after_create_making_history(changeset)
+    update_entity_imported_from_feeds(changeset)
+  end
+  def before_update_making_history(changeset)
+    update_entity_imported_from_feeds(changeset)
+  end
+
   def self.before_create_making_history(new_model, changeset)
     route = Route.find_by_onestop_id!(new_model.traversed_by)
     new_model.route = route
     self.existing_before_create_making_history(new_model, changeset)
   end
+
   # borrowed from schedule_stop_pair.rb
   def self.find_by_attributes(attrs = {})
     if attrs[:id].present?

diff --git a/app/models/stop.rb b/app/models/stop.rb
@@ -81,7 +81,9 @@ def csv_row_values
       :identified_by,
       :not_identified_by,
       :includes_stop_transfers,
-      :does_not_include_stop_transfers
+      :does_not_include_stop_transfers,
+      :add_imported_from_feeds,
+      :not_imported_from_feeds
     ],
     protected_attributes: [
       :identifiers,
@@ -97,12 +99,14 @@ def csv_row_values
 
   def after_create_making_history(changeset)
     super(changeset)
+    update_entity_imported_from_feeds(changeset)
     update_served_by(changeset)
     update_includes_stop_transfers(changeset)
     update_does_not_include_stop_transfers(changeset)
   end
   def before_update_making_history(changeset)
     super(changeset)
+    update_entity_imported_from_feeds(changeset)
     update_served_by(changeset)
     update_includes_stop_transfers(changeset)
     update_does_not_include_stop_transfers(changeset)

diff --git a/app/models/stop_egress.rb b/app/models/stop_egress.rb
@@ -42,7 +42,9 @@ class StopEgress < Stop
       :not_identified_by,
       :parent_stop_onestop_id,
       :includes_stop_transfers,
-      :does_not_include_stop_transfers
+      :does_not_include_stop_transfers,
+      :add_imported_from_feeds,
+      :not_imported_from_feeds
     ],
     protected_attributes: [
       :identifiers,

diff --git a/app/models/stop_platform.rb b/app/models/stop_platform.rb
@@ -42,7 +42,9 @@ class StopPlatform < Stop
       :not_identified_by,
       :parent_stop_onestop_id,
       :includes_stop_transfers,
-      :does_not_include_stop_transfers
+      :does_not_include_stop_transfers,
+      :add_imported_from_feeds,
+      :not_imported_from_feeds
     ],
     protected_attributes: [
       :identifiers,

diff --git a/app/serializers/entity_serializer.rb b/app/serializers/entity_serializer.rb
@@ -1,13 +1,8 @@
 class EntitySerializer < ApplicationSerializer
   attributes :identifiers,
-             :imported_from_feed_onestop_ids,
-             :imported_from_feed_version_sha1s
+             :imported_from_feeds
 
-  def imported_from_feed_onestop_ids
-    object.imported_from_feeds.map(&:onestop_id).uniq
-  end
-
-  def imported_from_feed_version_sha1s
-    object.imported_from_feed_versions.map(&:sha1).uniq
+  def imported_from_feeds
+    object.entities_imported_from_feed.map { |eiff| {feed_onestop_id: eiff.feed.try(:onestop_id), feed_version_sha1: eiff.feed_version.try(:sha1), gtfs_id: eiff.gtfs_id} }
   end
 end