From 58edbeefdb695fcb94f01d502c0ef32395c7d8d2 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 29 Nov 2016 12:53:17 -0800 Subject: [PATCH 1/8] Migration: add FeedVersion earliest_calendar_date / latest_calendar_date index --- .../20161129205145_add_calendar_indexes_to_feed_versions.rb | 6 ++++++ db/schema.rb | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20161129205145_add_calendar_indexes_to_feed_versions.rb diff --git a/db/migrate/20161129205145_add_calendar_indexes_to_feed_versions.rb b/db/migrate/20161129205145_add_calendar_indexes_to_feed_versions.rb new file mode 100644 index 000000000..a5cedd42d --- /dev/null +++ b/db/migrate/20161129205145_add_calendar_indexes_to_feed_versions.rb @@ -0,0 +1,6 @@ +class AddCalendarIndexesToFeedVersions < ActiveRecord::Migration + def change + add_index :feed_versions, :earliest_calendar_date + add_index :feed_versions, :latest_calendar_date + end +end diff --git a/db/schema.rb b/db/schema.rb index bfacf75e0..b6d267f86 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20161103231227) do +ActiveRecord::Schema.define(version: 20161129205145) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -304,6 +304,7 @@ t.datetime "created_at" t.datetime "updated_at" t.integer "feed_version_id" + t.string "gtfs_id" end add_index "entities_imported_from_feed", ["entity_type", "entity_id"], name: "index_entities_imported_from_feed_on_entity_type_and_entity_id", using: :btree @@ -365,7 +366,9 @@ t.string "md5_raw" end + add_index "feed_versions", ["earliest_calendar_date"], name: "index_feed_versions_on_earliest_calendar_date", using: :btree add_index "feed_versions", ["feed_type", "feed_id"], name: "index_feed_versions_on_feed_type_and_feed_id", using: :btree + add_index "feed_versions", ["latest_calendar_date"], name: "index_feed_versions_on_latest_calendar_date", using: :btree create_table "issues", force: :cascade do |t| t.integer "created_by_changeset_id" From 010a66c14110384d14c2d1849e7973e2c9b90fc0 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 29 Nov 2016 12:53:55 -0800 Subject: [PATCH 2/8] Annotation updates --- app/models/feed_version.rb | 4 +++- app/serializers/feed_version_serializer.rb | 4 +++- spec/factories/feed_version_factory.rb | 4 +++- spec/models/feed_version_spec.rb | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/app/models/feed_version.rb b/app/models/feed_version.rb index b49560840..9bd42c4a5 100644 --- a/app/models/feed_version.rb +++ b/app/models/feed_version.rb @@ -23,7 +23,9 @@ # # Indexes # -# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_earliest_calendar_date (earliest_calendar_date) +# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_latest_calendar_date (latest_calendar_date) # class FeedVersion < ActiveRecord::Base diff --git a/app/serializers/feed_version_serializer.rb b/app/serializers/feed_version_serializer.rb index 10b25e6e1..5f50a1270 100644 --- a/app/serializers/feed_version_serializer.rb +++ b/app/serializers/feed_version_serializer.rb @@ -23,7 +23,9 @@ # # Indexes # -# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_earliest_calendar_date (earliest_calendar_date) +# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_latest_calendar_date (latest_calendar_date) # class FeedVersionSerializer < ApplicationSerializer diff --git a/spec/factories/feed_version_factory.rb b/spec/factories/feed_version_factory.rb index 0063b06a0..86ff2367f 100644 --- a/spec/factories/feed_version_factory.rb +++ b/spec/factories/feed_version_factory.rb @@ -23,7 +23,9 @@ # # Indexes # -# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_earliest_calendar_date (earliest_calendar_date) +# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_latest_calendar_date (latest_calendar_date) # FactoryGirl.define do diff --git a/spec/models/feed_version_spec.rb b/spec/models/feed_version_spec.rb index afa20c766..d14425b66 100644 --- a/spec/models/feed_version_spec.rb +++ b/spec/models/feed_version_spec.rb @@ -23,7 +23,9 @@ # # Indexes # -# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_earliest_calendar_date (earliest_calendar_date) +# index_feed_versions_on_feed_type_and_feed_id (feed_type,feed_id) +# index_feed_versions_on_latest_calendar_date (latest_calendar_date) # describe FeedVersion do From 75d046ebf95560669a0cb7813dad22af4f7e0b8b Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 29 Nov 2016 12:54:07 -0800 Subject: [PATCH 3/8] FeedVersion calendar scopes --- app/models/feed_version.rb | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/models/feed_version.rb b/app/models/feed_version.rb index 9bd42c4a5..4a2e637b3 100644 --- a/app/models/feed_version.rb +++ b/app/models/feed_version.rb @@ -52,6 +52,22 @@ class FeedVersion < ActiveRecord::Base joins('INNER JOIN current_feeds ON feed_versions.id = current_feeds.active_feed_version_id') } + scope :calendar_coverage_begins_at_or_before, -> (date) { + date = date.is_a?(Date) ? date : Date.parse(date) + where('earliest_calendar_date <= ?', date) + } + + scope :calendar_coverage_begins_at_or_after, -> (date) { + date = date.is_a?(Date) ? date : Date.parse(date) + where('earliest_calendar_date >= ?', date) + } + + scope :calendar_coverage_includes, -> (date) { + date = date.is_a?(Date) ? date : Date.parse(date) + where('earliest_calendar_date <= ?', date) + .where('latest_calendar_date >= ?', date) + } + def succeeded(timestamp) self.update(imported_at: timestamp) self.feed.update(last_imported_at: self.imported_at) From e649aa8ee99042b8e6680d6210b3122c7d5fb352 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 29 Nov 2016 14:02:44 -0800 Subject: [PATCH 4/8] rename to where_ --- app/models/feed_version.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/feed_version.rb b/app/models/feed_version.rb index 4a2e637b3..ffb329dcf 100644 --- a/app/models/feed_version.rb +++ b/app/models/feed_version.rb @@ -52,17 +52,17 @@ class FeedVersion < ActiveRecord::Base joins('INNER JOIN current_feeds ON feed_versions.id = current_feeds.active_feed_version_id') } - scope :calendar_coverage_begins_at_or_before, -> (date) { + scope :where_calendar_coverage_begins_at_or_before, -> (date) { date = date.is_a?(Date) ? date : Date.parse(date) where('earliest_calendar_date <= ?', date) } - scope :calendar_coverage_begins_at_or_after, -> (date) { + scope :where_calendar_coverage_begins_at_or_after, -> (date) { date = date.is_a?(Date) ? date : Date.parse(date) where('earliest_calendar_date >= ?', date) } - scope :calendar_coverage_includes, -> (date) { + scope :where_calendar_coverage_includes, -> (date) { date = date.is_a?(Date) ? date : Date.parse(date) where('earliest_calendar_date <= ?', date) .where('latest_calendar_date >= ?', date) From 140159265fd235de25b754fb77bea84a54ba1dbd Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 29 Nov 2016 14:02:47 -0800 Subject: [PATCH 5/8] FeedVersions controller: add calendar coverage queries --- .../api/v1/feed_versions_controller.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/app/controllers/api/v1/feed_versions_controller.rb b/app/controllers/api/v1/feed_versions_controller.rb index 4c102317a..317d52aea 100644 --- a/app/controllers/api/v1/feed_versions_controller.rb +++ b/app/controllers/api/v1/feed_versions_controller.rb @@ -59,6 +59,24 @@ def index @feed_versions = @feed_versions.where(sha1: sha1s) end + if params[:calendar_coverage_begins_at_or_before].present? + @feed_versions = @feed_versions.where_calendar_coverage_begins_at_or_before( + params[:calendar_coverage_begins_at_or_before] + ) + end + + if params[:calendar_coverage_begins_at_or_after].present? + @feed_versions = @feed_versions.where_calendar_coverage_begins_at_or_after( + params[:calendar_coverage_begins_at_or_after] + ) + end + + if params[:calendar_coverage_includes].present? + @feed_versions = @feed_versions.where_calendar_coverage_includes( + params[:calendar_coverage_includes] + ) + end + if params[:feed_onestop_id].present? feed_onestop_ids = params[:feed_onestop_id].split(',') @feed_versions = @feed_versions.where(feed: Feed.where(onestop_id: feed_onestop_ids)) From 0a041a0676328844b1aeff3bedd6d2f23c738df1 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Tue, 29 Nov 2016 14:12:24 -0800 Subject: [PATCH 6/8] FeedVersion spec: where_calendar_coverage_begins_at_or_before --- spec/models/feed_version_spec.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/spec/models/feed_version_spec.rb b/spec/models/feed_version_spec.rb index d14425b66..d7ded924c 100644 --- a/spec/models/feed_version_spec.rb +++ b/spec/models/feed_version_spec.rb @@ -29,6 +29,16 @@ # describe FeedVersion do + context '.where_calendar_coverage_begins_at_or_before' do + it 'finds FeedVersions with coverage on or before a date' do + date = '2016-01-01' + fv1 = create(:feed_version, earliest_calendar_date: '2016-01-01') + fv2 = create(:feed_version, earliest_calendar_date: '2016-02-01') + fv3 = create(:feed_version, earliest_calendar_date: '2016-03-01') + expect(FeedVersion.where_calendar_coverage_begins_at_or_before('2016-02-01')).to match_array([fv1, fv2]) + end + end + context '#compute_and_set_hashes' do it 'computes file hashes' do feed_version = create(:feed_version_bart) From 34e4f242a25ccd25ae7cf797c88209aa0e242f1b Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Fri, 2 Dec 2016 11:50:59 -0800 Subject: [PATCH 7/8] Additional specs --- spec/models/feed_version_spec.rb | 48 +++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/spec/models/feed_version_spec.rb b/spec/models/feed_version_spec.rb index d7ded924c..d3f84366a 100644 --- a/spec/models/feed_version_spec.rb +++ b/spec/models/feed_version_spec.rb @@ -29,13 +29,47 @@ # describe FeedVersion do - context '.where_calendar_coverage_begins_at_or_before' do - it 'finds FeedVersions with coverage on or before a date' do - date = '2016-01-01' - fv1 = create(:feed_version, earliest_calendar_date: '2016-01-01') - fv2 = create(:feed_version, earliest_calendar_date: '2016-02-01') - fv3 = create(:feed_version, earliest_calendar_date: '2016-03-01') - expect(FeedVersion.where_calendar_coverage_begins_at_or_before('2016-02-01')).to match_array([fv1, fv2]) + context 'calendar scopes' do + before(:each) do + @fv1 = create(:feed_version, earliest_calendar_date: '2016-01-01', latest_calendar_date: '2017-01-01') + @fv2 = create(:feed_version, earliest_calendar_date: '2016-02-01', latest_calendar_date: '2017-02-01') + @fv3 = create(:feed_version, earliest_calendar_date: '2016-03-01', latest_calendar_date: '2017-03-01') + end + + context '.where_calendar_coverage_begins_at_or_before' do + it 'finds FeedVersions with coverage before a date' do + expect(FeedVersion.where_calendar_coverage_begins_at_or_before('2016-04-15')).to match_array([@fv1, @fv2, @fv3]) + end + it 'finds FeedVersions with coverage on or before a date' do + expect(FeedVersion.where_calendar_coverage_begins_at_or_before('2016-02-01')).to match_array([@fv1, @fv2]) + end + it 'finds FeedVersions with coverage on a date' do + expect(FeedVersion.where_calendar_coverage_begins_at_or_before('2016-01-01')).to match_array([@fv1]) + end + end + + context '.where_calendar_coverage_begins_at_or_after' do + it 'finds FeedVersions with coverage after a date' do + expect(FeedVersion.where_calendar_coverage_begins_at_or_after('2015-12-01')).to match_array([@fv1, @fv2, @fv3]) + end + it 'finds FeedVersions with coverage on or after a date' do + expect(FeedVersion.where_calendar_coverage_begins_at_or_after('2016-02-01')).to match_array([@fv2, @fv3]) + end + it 'finds FeedVersions with coverage on a date' do + expect(FeedVersion.where_calendar_coverage_begins_at_or_after('2016-03-01')).to match_array([@fv3]) + end + end + + context '.where_calendar_coverage_includes' do + it 'finds FeedVersions with coverage including a date' do + expect(FeedVersion.where_calendar_coverage_includes('2016-04-01')).to match_array([@fv1, @fv2, @fv3]) + end + it 'finds FeedVersions with coverage including, inclusive' do + expect(FeedVersion.where_calendar_coverage_includes('2016-02-01')).to match_array([@fv1, @fv2]) + end + it 'excludes FeedVersions outside coverage range' do + expect(FeedVersion.where_calendar_coverage_includes('2017-01-15')).to match_array([@fv2, @fv3]) + end end end From 7d4910c931e735fa9313de11ad67531cb17e6c54 Mon Sep 17 00:00:00 2001 From: Ian Rees Date: Fri, 2 Dec 2016 15:08:35 -0800 Subject: [PATCH 8/8] FeedVersion controller specs --- .../api/v1/feed_versions_controller_spec.rb | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/spec/controllers/api/v1/feed_versions_controller_spec.rb b/spec/controllers/api/v1/feed_versions_controller_spec.rb index c1a31d286..36077b1ff 100644 --- a/spec/controllers/api/v1/feed_versions_controller_spec.rb +++ b/spec/controllers/api/v1/feed_versions_controller_spec.rb @@ -14,6 +14,33 @@ }) end + it 'calendar_coverage_begins_at_or_before' do + fv1 = create(:feed_version, earliest_calendar_date: '2016-01-01', latest_calendar_date: '2017-01-01') + fv2 = create(:feed_version, earliest_calendar_date: '2016-02-01', latest_calendar_date: '2017-02-01') + get :index, calendar_coverage_begins_at_or_before: '2016-01-01' + expect_json({feed_versions: -> (feed_versions) { + expect(feed_versions.map { |fv| fv[:sha1]}).to match_array([fv1.sha1]) + }}) + end + + it 'calendar_coverage_begins_at_or_after' do + fv1 = create(:feed_version, earliest_calendar_date: '2016-01-01', latest_calendar_date: '2017-01-01') + fv2 = create(:feed_version, earliest_calendar_date: '2016-02-01', latest_calendar_date: '2017-02-01') + get :index, calendar_coverage_begins_at_or_after: '2016-02-01' + expect_json({feed_versions: -> (feed_versions) { + expect(feed_versions.map { |fv| fv[:sha1]}).to match_array([fv2.sha1]) + }}) + end + + it 'calendar_coverage_includes' do + fv1 = create(:feed_version, earliest_calendar_date: '2016-01-01', latest_calendar_date: '2017-01-01') + fv2 = create(:feed_version, earliest_calendar_date: '2016-02-01', latest_calendar_date: '2017-02-01') + get :index, calendar_coverage_includes: '2017-01-15' + expect_json({feed_versions: -> (feed_versions) { + expect(feed_versions.map { |fv| fv[:sha1]}).to match_array([fv2.sha1]) + }}) + end + it 'filters by SHA1 hash' do create_list(:feed_version, 2) sha1 = FeedVersion.first.sha1