Production release 4.9.14

CHANGELOG.md

@@ -2,10 +2,28 @@
 
 ## [Unreleased](https://github.com/transitland/transitland-datastore/tree/HEAD)
 
-[Full Changelog](https://github.com/transitland/transitland-datastore/compare/4.9.12...HEAD)
+[Full Changelog](https://github.com/transitland/transitland-datastore/compare/4.9.13...HEAD)
 
 **Merged pull requests:**
 
+- Production release 4.9.13 [\#858](https://github.com/transitland/transitland-datastore/pull/858) ([irees](https://github.com/irees))
+
+## [4.9.13](https://github.com/transitland/transitland-datastore/tree/4.9.13) (2016-11-21)
+[Full Changelog](https://github.com/transitland/transitland-datastore/compare/4.9.12...4.9.13)
+
+**Implemented enhancements:**
+
+- When bulk deactivating issues, make sure it's done asynchronously [\#787](https://github.com/transitland/transitland-datastore/issues/787)
+
+**Closed issues:**
+
+- Override stops controller to use 'stops' as root [\#862](https://github.com/transitland/transitland-datastore/issues/862)
+- Save Feed fetch errors as issues [\#820](https://github.com/transitland/transitland-datastore/issues/820)
+
+**Merged pull requests:**
+
+- Stop serializer root fix [\#861](https://github.com/transitland/transitland-datastore/pull/861) ([irees](https://github.com/irees))
+- feed fetch errors create issues [\#844](https://github.com/transitland/transitland-datastore/pull/844) ([doublestranded](https://github.com/doublestranded))
 - Production release 4.9.12 [\#843](https://github.com/transitland/transitland-datastore/pull/843) ([irees](https://github.com/irees))
 
 ## [4.9.12](https://github.com/transitland/transitland-datastore/tree/4.9.12) (2016-11-14)

Gemfile.lock

@@ -65,7 +65,7 @@ GEM
       activemodel (= 4.2.7.1)
       activesupport (= 4.2.7.1)
       arel (~> 6.0)
-    activerecord-import (0.16.1)
+    activerecord-import (0.16.2)
       activerecord (>= 3.2)
     activerecord-postgis-adapter (3.1.4)
       activerecord (~> 4.2)
@@ -88,12 +88,14 @@ GEM
       rake (>= 10.4, < 12.0)
     arel (6.0.3)
     ast (2.3.0)
-    aws-sdk (2.6.12)
-      aws-sdk-resources (= 2.6.12)
-    aws-sdk-core (2.6.12)
+    aws-sdk (2.6.32)
+      aws-sdk-resources (= 2.6.32)
+    aws-sdk-core (2.6.32)
+      aws-sigv4 (~> 1.0)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.6.12)
-      aws-sdk-core (= 2.6.12)
+    aws-sdk-resources (2.6.32)
+      aws-sdk-core (= 2.6.32)
+    aws-sigv4 (1.0.0)
     bcrypt (3.1.11)
     better_errors (2.1.1)
       coderay (>= 1.0.0)
@@ -111,7 +113,7 @@ GEM
     chronic (0.10.2)
     coderay (1.1.1)
     concurrent-ruby (1.0.2)
-    connection_pool (2.2.0)
+    connection_pool (2.2.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     database_cleaner (1.5.3)
@@ -124,7 +126,7 @@ GEM
       warden (~> 1.2.3)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    domain_name (0.5.20160826)
+    domain_name (0.5.20161129)
       unf (>= 0.0.5, < 1.0.0)
     email_validator (1.6.0)
       activemodel
@@ -137,12 +139,12 @@ GEM
     factory_girl_rails (4.7.0)
       factory_girl (~> 4.7.0)
       railties (>= 3.0.0)
-    faraday (0.9.2)
+    faraday (0.10.0)
       multipart-post (>= 1.2, < 3)
-    ffaker (2.2.0)
+    ffaker (2.3.0)
     figaro (1.1.1)
       thor (~> 0.14)
-    fog-aws (0.12.0)
+    fog-aws (0.13.0)
       fog-core (~> 1.38)
       fog-json (~> 1.0)
       fog-xml (~> 0.1)
@@ -162,7 +164,7 @@ GEM
     formatador (0.2.5)
     globalid (0.3.7)
       activesupport (>= 4.1.0)
-    hashdiff (0.3.0)
+    hashdiff (0.3.1)
     http-cookie (1.0.3)
       domain_name (~> 0.5)
     i18n (0.7.0)
@@ -179,7 +181,7 @@ GEM
     jsonapi-renderer (0.1.1.beta1)
     kgio (2.10.0)
     logstash-event (1.2.02)
-    logstasher (1.0.1)
+    logstasher (1.1.0)
       activerecord (>= 4.0)
       activesupport (>= 4.0)
       logstash-event (~> 1.2.0)
@@ -191,25 +193,25 @@ GEM
     marginalia (1.4.0)
       actionpack (>= 2.3)
       activerecord (>= 2.3)
-    memory_profiler (0.9.6)
+    memory_profiler (0.9.7)
     method_source (0.8.2)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
     mini_portile2 (2.1.0)
-    minitest (5.9.1)
+    minitest (5.10.1)
     multi_json (1.12.1)
     multipart-post (2.0.0)
     netrc (0.11.0)
     nokogiri (1.6.8.1)
       mini_portile2 (~> 2.1.0)
-    oj (2.17.5)
+    oj (2.18.0)
     oj_mimic_json (1.0.1)
     orm_adapter (0.5.0)
-    parser (2.3.1.4)
+    parser (2.3.3.1)
       ast (~> 2.2)
     pg (0.19.0)
-    pghero (1.6.0)
+    pghero (1.6.2)
       activerecord
     polyamorous (1.1.0)
       activerecord (>= 3.0)
@@ -218,7 +220,7 @@ GEM
       coderay (~> 1.1.0)
       method_source (~> 0.8.1)
       slop (~> 3.4)
-    pry-byebug (3.4.0)
+    pry-byebug (3.4.1)
       byebug (~> 9.0)
       pry (~> 0.10)
     pry-rails (0.3.4)
@@ -229,7 +231,7 @@ GEM
     pry-stack_explorer (0.4.9.2)
       binding_of_caller (>= 0.7)
       pry (>= 0.9.11)
-    rack (1.6.4)
+    rack (1.6.5)
     rack-cors (0.4.0)
     rack-protection (1.5.3)
       rack
@@ -268,17 +270,17 @@ GEM
     raindrops (0.17.0)
     rake (11.3.0)
     rcsv (0.3.1)
-    redis (3.3.1)
-    redis-actionpack (5.0.0)
-      actionpack (>= 4.0.0, < 6)
-      redis-rack (~> 2.0.0.pre)
-      redis-store (~> 1.2.0.pre)
+    redis (3.3.2)
+    redis-actionpack (5.0.1)
+      actionpack (>= 4.0, < 6)
+      redis-rack (>= 1, < 3)
+      redis-store (>= 1.1.0, < 1.4.0)
     redis-activesupport (5.0.1)
       activesupport (>= 3, < 6)
       redis-store (~> 1.2.0)
-    redis-rack (2.0.0.pre)
-      rack (> 1.5, < 3)
-      redis-store (~> 1.2.0.pre)
+    redis-rack (1.6.0)
+      rack (~> 1.5)
+      redis-store (~> 1.2.0)
     redis-rails (5.0.1)
       redis-actionpack (~> 5.0.0)
       redis-activesupport (~> 5.0.0)
@@ -325,21 +327,21 @@ GEM
     rspec_junit_formatter (0.2.2)
       builder (< 4)
       rspec-core (>= 2, < 4, != 2.12.0)
-    rubocop (0.44.1)
+    rubocop (0.46.0)
       parser (>= 2.3.1.1, < 3.0)
       powerpack (~> 0.1)
       rainbow (>= 1.99.1, < 3.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (~> 1.0, >= 1.0.1)
-    rubocop-rspec (1.7.0)
+    rubocop-rspec (1.8.0)
       rubocop (>= 0.42.0)
     ruby-graphviz (1.2.2)
     ruby-progressbar (1.8.1)
     rubyzip (1.2.0)
     safe_yaml (1.0.4)
-    sentry-raven (2.1.2)
-      faraday (>= 0.7.6, < 0.10.x)
-    sidekiq (4.2.3)
+    sentry-raven (2.2.0)
+      faraday (>= 0.7.6, < 1.0)
+    sidekiq (4.2.7)
       concurrent-ruby (~> 1.0)
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
@@ -373,7 +375,7 @@ GEM
       activesupport (>= 3.0)
       polyamorous (~> 1.1.0)
     text (1.3.1)
-    thor (0.19.1)
+    thor (0.19.4)
     thread_safe (0.3.5)
     tilt (2.0.5)
     timecop (0.8.1)
@@ -383,14 +385,14 @@ GEM
       unf_ext
     unf_ext (0.0.7.2)
     unicode-display_width (1.1.1)
-    unicorn (5.1.0)
+    unicorn (5.2.0)
       kgio (~> 2.6)
       raindrops (~> 0.7)
     uniform_notifier (1.10.0)
     vcr (3.0.3)
     warden (1.2.6)
       rack (>= 1.0)
-    webmock (2.1.0)
+    webmock (2.3.1)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
@@ -464,4 +466,4 @@ DEPENDENCIES
   whenever
 
 BUNDLED WITH
-   1.13.2
+   1.13.6

app/controllers/api/v1/feed_versions_controller.rb

@@ -59,6 +59,24 @@ def index
       @feed_versions = @feed_versions.where(sha1: sha1s)
     end
 
+    if params[:calendar_coverage_begins_at_or_before].present?
+      @feed_versions = @feed_versions.where_calendar_coverage_begins_at_or_before(
+        params[:calendar_coverage_begins_at_or_before]
+      )
+    end
+
+    if params[:calendar_coverage_begins_at_or_after].present?
+      @feed_versions = @feed_versions.where_calendar_coverage_begins_at_or_after(
+        params[:calendar_coverage_begins_at_or_after]
+      )
+    end
+
+    if params[:calendar_coverage_includes].present?
+      @feed_versions = @feed_versions.where_calendar_coverage_includes(
+        params[:calendar_coverage_includes]
+      )
+    end
+
     if params[:feed_onestop_id].present?
       feed_onestop_ids = params[:feed_onestop_id].split(',')
       @feed_versions = @feed_versions.where(feed: Feed.where(onestop_id: feed_onestop_ids))

app/models/feed.rb

@@ -54,13 +54,16 @@ class Feed < BaseFeed
   include HasTags
   include UpdatedSince
   include HasAGeographicGeometry
+  include IsAnEntityWithIssues
 
   has_many :feed_versions, -> { order 'created_at DESC' }, dependent: :destroy, as: :feed
   has_many :feed_version_imports, -> { order 'created_at DESC' }, through: :feed_versions
   belongs_to :active_feed_version, class_name: 'FeedVersion'
 
   has_many :operators_in_feed
-  has_many :operators, through: :operators_in_feed
+  has_many :operators, -> { distinct }, through: :operators_in_feed
+
+  has_many :issues, through: :entities_with_issues
 
   has_many :entities_imported_from_feed
   has_many :imported_operators, through: :entities_imported_from_feed, source: :entity, source_type: 'Operator'
@@ -176,7 +179,11 @@ def after_create_making_history(changeset)
   def before_update_making_history(changeset)
     (self.includes_operators || []).each do |included_operator|
       operator = Operator.find_by!(onestop_id: included_operator[:operator_onestop_id])
-      existing_relationship = OperatorInFeed.find_by(operator: operator, feed: self)
+      existing_relationship = OperatorInFeed.find_by(
+        operator: operator,
+        gtfs_agency_id: included_operator[:gtfs_agency_id],
+        feed: self
+      )
       if existing_relationship
           existing_relationship.update_making_history(
             changeset: changeset,
@@ -199,7 +206,11 @@ def before_update_making_history(changeset)
     end
     (self.does_not_include_operators || []).each do |not_included_operator|
       operator = Operator.find_by!(onestop_id: not_included_operator[:operator_onestop_id])
-      existing_relationship = OperatorInFeed.find_by(operator: operator, feed: self)
+      existing_relationship = OperatorInFeed.find_by(
+        operator: operator,
+        gtfs_agency_id: not_included_operator[:gtfs_agency_id],
+        feed: self
+      )
       if existing_relationship
         existing_relationship.destroy_making_history(changeset: changeset)
       end

app/models/feed_version.rb

@@ -20,10 +20,13 @@
 #  file_raw               :string
 #  sha1_raw               :string
 #  md5_raw                :string
+#  file_feedvalidator     :string
 #
 # Indexes
 #
-#  index_feed_versions_on_feed_type_and_feed_id  (feed_type,feed_id)
+#  index_feed_versions_on_earliest_calendar_date  (earliest_calendar_date)
+#  index_feed_versions_on_feed_type_and_feed_id   (feed_type,feed_id)
+#  index_feed_versions_on_latest_calendar_date    (latest_calendar_date)
 #
 
 class FeedVersion < ActiveRecord::Base
@@ -40,6 +43,7 @@ class FeedVersion < ActiveRecord::Base
 
   mount_uploader :file, FeedVersionUploader
   mount_uploader :file_raw, FeedVersionUploaderRaw
+  mount_uploader :file_feedvalidator, FeedVersionUploaderFeedvalidator
 
   validates :sha1, presence: true, uniqueness: true
   validates :feed, presence: true
@@ -50,6 +54,22 @@ class FeedVersion < ActiveRecord::Base
     joins('INNER JOIN current_feeds ON feed_versions.id = current_feeds.active_feed_version_id')
   }
 
+  scope :where_calendar_coverage_begins_at_or_before, -> (date) {
+    date = date.is_a?(Date) ? date : Date.parse(date)
+    where('earliest_calendar_date <= ?', date)
+  }
+
+  scope :where_calendar_coverage_begins_at_or_after, -> (date) {
+    date = date.is_a?(Date) ? date : Date.parse(date)
+    where('earliest_calendar_date >= ?', date)
+  }
+
+  scope :where_calendar_coverage_includes, -> (date) {
+    date = date.is_a?(Date) ? date : Date.parse(date)
+    where('earliest_calendar_date <= ?', date)
+      .where('latest_calendar_date >= ?', date)
+  }
+
   def succeeded(timestamp)
     self.update(imported_at: timestamp)
     self.feed.update(last_imported_at: self.imported_at)
@@ -102,6 +122,13 @@ def download_url
     end
   end
 
+  def feedvalidator_url
+    if self.try(:file_feedvalidator).try(:url)
+      # we don't want to include any query parameters
+      self.file_feedvalidator.url.split('?').first
+    end
+  end
+
   private
 
   def compute_and_set_hashes

app/models/issue.rb

@@ -31,6 +31,7 @@ class Issue < ActiveRecord::Base
   enumerize :issue_type,
             in: ['stop_position_inaccurate',
                  'stop_rsp_distance_gap',
+                 'missing_stop_conflation_result',
                  'distance_calculation_inaccurate',
                  'rsp_line_inaccurate',
                  'route_color',

app/models/json_schemas/feed.json

@@ -41,7 +41,7 @@
       "uniqueItems": true,
       "items": {
         "type": "object",
-        "required": ["operatorOnestopId"],
+        "required": ["operatorOnestopId", "gtfsAgencyId"],
         "operatorOnestopId": {
           "type": "string",
           "format": "operator-onestop-id"