Production release 74

CHANGELOG.md

@@ -2,15 +2,35 @@
 
 ## [Unreleased](https://github.com/transitland/transitland-datastore/tree/HEAD)
 
-[Full Changelog](https://github.com/transitland/transitland-datastore/compare/72...HEAD)
+[Full Changelog](https://github.com/transitland/transitland-datastore/compare/73...HEAD)
+
+**Merged pull requests:**
+
+- Production release 73 [\#1140](https://github.com/transitland/transitland-datastore/pull/1140) ([irees](https://github.com/irees))
+
+## [73](https://github.com/transitland/transitland-datastore/tree/73) (2017-08-03)
+[Full Changelog](https://github.com/transitland/transitland-datastore/compare/72...73)
 
 **Closed issues:**
 
+- RSP: Use stop centroids [\#1148](https://github.com/transitland/transitland-datastore/issues/1148)
+- Use 64 bit ID for SSP [\#1141](https://github.com/transitland/transitland-datastore/issues/1141)
 - SSP: Query by operator\_onestop\_id is slow [\#1138](https://github.com/transitland/transitland-datastore/issues/1138)
+- Polygons \(not just points\) for StopStation geometries [\#826](https://github.com/transitland/transitland-datastore/issues/826)
+- consistent precision for geometries [\#362](https://github.com/transitland/transitland-datastore/issues/362)
 
 **Merged pull requests:**
 
+- Gracefully handle references to missing stop or route [\#1153](https://github.com/transitland/transitland-datastore/pull/1153) ([irees](https://github.com/irees))
+- Stop: fallback to stop\_id if stop\_name is not present [\#1151](https://github.com/transitland/transitland-datastore/pull/1151) ([irees](https://github.com/irees))
+- Stop serializer: use geometry\_centroid not centroid [\#1150](https://github.com/transitland/transitland-datastore/pull/1150) ([irees](https://github.com/irees))
+- Include geometry\_reversegeo and centroid in Stop and Station serializers [\#1149](https://github.com/transitland/transitland-datastore/pull/1149) ([irees](https://github.com/irees))
+- RSP: Fixes for stops with polygon geometries [\#1147](https://github.com/transitland/transitland-datastore/pull/1147) ([irees](https://github.com/irees))
+- Tidy up: schema & annotation update [\#1146](https://github.com/transitland/transitland-datastore/pull/1146) ([irees](https://github.com/irees))
+- Cleanup: Alter SSP id to bigserial [\#1145](https://github.com/transitland/transitland-datastore/pull/1145) ([irees](https://github.com/irees))
+- update gems [\#1143](https://github.com/transitland/transitland-datastore/pull/1143) ([drewda](https://github.com/drewda))
 - Production release 72 [\#1128](https://github.com/transitland/transitland-datastore/pull/1128) ([irees](https://github.com/irees))
+- Geometry validation & station polygons [\#904](https://github.com/transitland/transitland-datastore/pull/904) ([irees](https://github.com/irees))
 
 ## [72](https://github.com/transitland/transitland-datastore/tree/72) (2017-07-20)
 [Full Changelog](https://github.com/transitland/transitland-datastore/compare/71...72)
@@ -21,7 +41,6 @@
 
 **Closed issues:**
 
-- No operators in feed issue: improve descriptive text [\#1132](https://github.com/transitland/transitland-datastore/issues/1132)
 - allow uploading of feed versions for an existing feed [\#1125](https://github.com/transitland/transitland-datastore/issues/1125)
 - Station hierarchy improvements [\#1066](https://github.com/transitland/transitland-datastore/issues/1066)
 - GTFS Station Egress: location\_type = 2 [\#643](https://github.com/transitland/transitland-datastore/issues/643)

Gemfile.lock

@@ -77,23 +77,23 @@ GEM
       activerecord (>= 3.2, < 6.0)
       rake (>= 10.4, < 13.0)
     arel (6.0.4)
-    aws-sdk (2.10.19)
-      aws-sdk-resources (= 2.10.19)
-    aws-sdk-core (2.10.19)
+    aws-sdk (2.10.25)
+      aws-sdk-resources (= 2.10.25)
+    aws-sdk-core (2.10.25)
       aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.10.19)
-      aws-sdk-core (= 2.10.19)
+    aws-sdk-resources (2.10.25)
+      aws-sdk-core (= 2.10.25)
     aws-sigv4 (1.0.1)
     bcrypt (3.1.11)
-    better_errors (2.1.1)
+    better_errors (2.3.0)
       coderay (>= 1.0.0)
-      erubis (>= 2.6.6)
+      erubi (>= 1.0.0)
       rack (>= 0.9.0)
     binding_of_caller (0.7.2)
       debug_inspector (>= 0.0.1)
     builder (3.2.3)
-    bullet (5.6.0)
+    bullet (5.6.1)
       activesupport (>= 3.0.0)
       uniform_notifier (~> 1.10.0)
     byebug (9.0.6)
@@ -127,8 +127,9 @@ GEM
       activemodel
     enumerize (2.1.2)
       activesupport (>= 3.2)
+    erubi (1.6.1)
     erubis (2.7.0)
-    excon (0.57.1)
+    excon (0.58.0)
     factory_girl (4.8.0)
       activesupport (>= 3.0.0)
     factory_girl_rails (4.8.0)
@@ -144,9 +145,9 @@ GEM
       fog-json (~> 1.0)
       fog-xml (~> 0.1)
       ipaddress (~> 0.8)
-    fog-core (1.44.3)
+    fog-core (1.45.0)
       builder
-      excon (~> 0.49)
+      excon (~> 0.58)
       formatador (~> 0.2)
     fog-json (1.0.2)
       fog-core (~> 1.0)
@@ -159,7 +160,7 @@ GEM
     formatador (0.2.5)
     globalid (0.4.0)
       activesupport (>= 4.2.0)
-    hashdiff (0.3.4)
+    hashdiff (0.3.5)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     i18n (0.8.6)
@@ -196,10 +197,10 @@ GEM
     netrc (0.11.0)
     nokogiri (1.8.0)
       mini_portile2 (~> 2.2.0)
-    oj (3.3.2)
+    oj (3.3.5)
     orm_adapter (0.5.0)
     pg (0.21.0)
-    pghero (1.7.0)
+    pghero (2.0.2)
       activerecord
     polyamorous (1.1.0)
       activerecord (>= 3.0)
@@ -254,7 +255,7 @@ GEM
       activesupport (= 4.2.9)
       rake (>= 0.8.7)
       thor (>= 0.18.1, < 2.0)
-    raindrops (0.18.0)
+    raindrops (0.19.0)
     rake (12.0.0)
     rcsv (0.3.1)
     redis (3.3.3)
@@ -300,7 +301,7 @@ GEM
     rspec-mocks (3.6.0)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.6.0)
-    rspec-rails (3.6.0)
+    rspec-rails (3.6.1)
       actionpack (>= 3.0)
       activesupport (>= 3.0)
       railties (>= 3.0)
@@ -319,7 +320,7 @@ GEM
     ruby-progressbar (1.8.1)
     rubyzip (1.2.1)
     safe_yaml (1.0.4)
-    sentry-raven (2.6.0)
+    sentry-raven (2.6.3)
       faraday (>= 0.7.6, < 1.0)
     sidekiq (5.0.4)
       concurrent-ruby (~> 1.0)
@@ -333,11 +334,11 @@ GEM
     sidekiq-unique-jobs (5.0.9)
       sidekiq (>= 4.0, <= 6.0)
       thor (~> 0)
-    simplecov (0.14.1)
+    simplecov (0.15.0)
       docile (~> 1.1.0)
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
-    simplecov-html (0.10.1)
+    simplecov-html (0.10.2)
     sinatra (1.4.8)
       rack (~> 1.5)
       rack-protection (~> 1.4)

app/controllers/api/v1/feeds_controller.rb

@@ -35,6 +35,7 @@ def download_latest_feed_version
 
   def index_query
     super
+    @collection = AllowFiltering.by_attribute_array(@collection, params, :name)
     @collection = AllowFiltering.by_attribute_array(@collection, params, :url, case_sensitive: true)
     @collection = AllowFiltering.by_attribute_since(@collection, params, :last_imported_since, :last_imported_at)
     if params[:latest_fetch_exception].present?
@@ -69,6 +70,11 @@ def index_includes
 
   def query_params
     super.merge({
+      name: {
+        desc: "Feed name",
+        type: "string",
+        array: true
+      },
       last_imported_since: {
         desc: "Last imported since",
         type: "datetime"

app/controllers/api/v1/operators_controller.rb

@@ -40,6 +40,16 @@ def index_query
     @collection = AllowFiltering.by_attribute_array(@collection, params, :timezone)
     @collection = AllowFiltering.by_attribute_array(@collection, params, :name)
     @collection = AllowFiltering.by_attribute_array(@collection, params, :short_name)
+
+    # with_feed
+    if params[:with_feed].present?
+      @collection = @collection.with_feed(Feed.find_by_onestop_ids!(AllowFiltering.param_as_array(params, :with_feed)))
+    end
+
+    # without_feed
+    if params[:without_feed].presence == 'true'
+      @collection = @collection.without_feed
+    end
   end
 
   def query_params
@@ -73,6 +83,15 @@ def query_params
         desc: "Operator timezone",
         type: "string",
         array: true
+      },
+      with_feed: {
+        desc: "Operators referenced by a Feed",
+        type: "string",
+        array: true
+      },
+      without_feed: {
+        desc: "Operators not referenced by any Feed",
+        type: "boolean",
       }
     })
   end

app/models/concerns/current_tracked_by_changeset.rb

@@ -22,6 +22,8 @@ def apply_change(changeset: nil, change: {}, action: nil, onestop_ids_to_merge:
         apply_change_destroy(changeset: changeset, change: change)
       when 'changeOnestopID'
         apply_change_change_onestop_id(changeset: changeset, change: change)
+      when 'changeStopType'
+        apply_change_change_stop_type(changeset: changeset, change: change)
       when 'merge'
         if onestop_ids_to_merge.nil?
           raise Changeset::Error.new(changeset: changeset, message: "Error: must provide an array of onestop ids to merge.")
@@ -64,6 +66,42 @@ def apply_change_change_onestop_id(changeset: nil, change: nil)
       end
     end
 
+    def apply_change_change_stop_type(changeset: nil, change: nil)
+      stop = Stop.find_by_onestop_id!(change[:onestop_id])
+      stop_type = change[:stop_type]
+
+      if stop_type == "Stop"
+        parent_stop = nil
+        new_onestop_id = change[:new_onestop_id]
+      elsif stop_type == "StopPlatform"
+        platform_name = change[:platform_name]
+        parent_stop = Stop.find_by_onestop_id!(change[:parent_stop_onestop_id])
+        new_onestop_id = "#{parent_stop.onestop_id}<#{platform_name}"
+        change[:new_onestop_id] = new_onestop_id
+      elsif stop_type == "StopEgress"
+        platform_name = change[:platform_name]
+        parent_stop = Stop.find_by_onestop_id!(change[:parent_stop_onestop_id])
+        new_onestop_id = "#{parent_stop.onestop_id}>#{platform_name}"
+        change[:new_onestop_id] = new_onestop_id
+      else
+        raise Changeset::Error.new(changeset: changeset, message: "unknown stop_type: #{stop_type}")
+      end
+
+      # Rename - change 1
+      apply_change_change_onestop_id(changeset: changeset, change: change)
+
+      # Record change_stop_type action - change 2
+      stop.reload
+      stop.update_making_history(changeset: changeset, new_attrs: {}, old_attrs: { action: 'change_stop_type' })
+
+      # Becomes
+      # Set parent_stop manually
+      stop.becomes(stop_type.constantize).update!(
+       type: (stop_type == 'Stop' ? nil : stop_type),
+       parent_stop: parent_stop
+      )
+    end
+
     def after_merge_onestop_ids(changeset)
       # this is available for overriding in models
       super(changeset) if defined?(super)

app/models/feed.rb

@@ -22,6 +22,7 @@
 #  license_attribution_text           :text
 #  active_feed_version_id             :integer
 #  edited_attributes                  :string           default([]), is an Array
+#  name                               :string
 #
 # Indexes
 #
@@ -57,7 +58,23 @@ class Feed < BaseFeed
   include IsAnEntityWithIssues
   include IsAnEntityImportedFromFeeds
 
-  has_many :feed_versions, -> { order 'created_at DESC' }, dependent: :destroy, as: :feed
+  include CanBeSerializedToCsv
+  def self.csv_column_names
+    [
+      'Onestop ID',
+      'Name',
+      'URL'
+    ]
+  end
+  def csv_row_values
+    [
+      onestop_id,
+      name,
+      url
+    ]
+  end
+
+  has_many :feed_versions, -> { order 'earliest_calendar_date' }, dependent: :destroy, as: :feed
   has_many :feed_version_imports, -> { order 'created_at DESC' }, through: :feed_versions
   belongs_to :active_feed_version, class_name: 'FeedVersion'
 
@@ -159,7 +176,7 @@ class Feed < BaseFeed
   })
 
   def update_associations(changeset)
-    (self.includes_operators || []).each do |included_operator|
+    (self.includes_operators || []).uniq.each do |included_operator|
       operator = Operator.find_by!(onestop_id: included_operator[:operator_onestop_id])
       existing_relationship = OperatorInFeed.find_by(
         operator: operator,
@@ -186,7 +203,7 @@ def update_associations(changeset)
         )
       end
     end
-    (self.does_not_include_operators || []).each do |not_included_operator|
+    (self.does_not_include_operators || []).uniq.each do |not_included_operator|
       operator = Operator.find_by!(onestop_id: not_included_operator[:operator_onestop_id])
       existing_relationship = OperatorInFeed.find_by(
         operator: operator,

app/models/json_schemas/changeset.json

@@ -78,7 +78,7 @@
         "properties": {
           "action": {
             "type": "string",
-            "enum": ["createUpdate", "destroy", "changeOnestopID", "merge"]
+            "enum": ["createUpdate", "destroy", "changeOnestopID", "changeStopType", "merge"]
           },
           "issuesResolved": {
             "type": "array",

app/models/json_schemas/feed.json

@@ -35,7 +35,6 @@
     "includesOperators": {
       "type": "array",
       "minItems": 1,
-      "uniqueItems": true,
       "items": {
         "type": "object",
         "required": ["operatorOnestopId", "gtfsAgencyId"],
@@ -51,7 +50,6 @@
     "doesNotIncludeOperators": {
       "type": "array",
       "minItems": 1,
-      "uniqueItems": true,
       "items": {
         "type": "object",
         "required": ["operatorOnestopId", "gtfsAgencyId"],

app/models/json_schemas/operator.json

@@ -33,30 +33,26 @@
     "addImportedFromFeeds": {
       "type": "array",
       "minItems": 1,
-      "uniqueItems": true,
       "items": {
         "$ref": "./imported_from_feed.json"
       }
     },
     "notImportedFromFeeds": {
       "type": "array",
       "minItems": 1,
-      "uniqueItems": true,
       "items": {
         "$ref": "./imported_from_feed.json"
       }
     },
     "serves": {
       "type": "array",
-      "uniqueItems": true,
       "items": {
         "type": "string",
         "format": "stop-onestop-id"
       }
     },
     "doesNotServe": {
       "type": "array",
-      "uniqueItems": true,
       "items": {
         "type": "string",
         "format": "stop-onestop-id"