Merge pull request #819 from transitland/feed-filter-by-import-status

Feed: filter by latest import status

app/controllers/api/v1/feeds_controller.rb

@@ -64,6 +64,10 @@ def index
       @feeds = @feeds.where_active_feed_version_import_level(params[:active_feed_version_import_level])
     end
 
+    if params[:latest_feed_version_import_status].present?
+      @feeds = @feeds.where_latest_feed_version_import_status(AllowFiltering.to_boolean(params[:latest_feed_version_import_status]))
+    end
+
     respond_to do |format|
       format.json do
         render paginated_json_collection(
@@ -77,10 +81,14 @@ def index
           params.slice(
             :tag_key,
             :tag_value,
+            :bbox,
             :last_imported_since,
             :active_feed_version_valid,
             :active_feed_version_expired,
-            :active_feed_version_update
+            :active_feed_version_update,
+            :active_feed_version_import_level,
+            :latest_feed_version_import_status,
+            :latest_fetch_exception
           )
         )
       end

app/controllers/concerns/allow_filtering.rb

@@ -115,7 +115,8 @@ def self.to_boolean(value)
       when false then false
       when 0 then false
       when '0' then false
-      when nil then false
+      when nil then nil
+      when 'null' then nil
     end
   end
 

app/models/feed.rb

@@ -105,7 +105,7 @@ class Feed < BaseFeed
   scope :where_active_feed_version_update, -> {
     # Find feeds that have a feed_version newer than
     #   the current active_feed_version
-    joins(p %{
+    joins(%{
       INNER JOIN (
         SELECT DISTINCT feed_versions.feed_id
         FROM feed_versions
@@ -121,6 +121,29 @@ class Feed < BaseFeed
     })
   }
 
+  scope :where_latest_feed_version_import_status, -> (import_status) {
+    # Get the highest fvi id (=~ created_at) for each feed,
+    # then filter by that fvi's success status.
+    joins(%{
+      INNER JOIN (
+        SELECT fv.feed_id, MAX(fvi.id) fvi_max_id
+        FROM feed_versions fv
+        INNER JOIN feed_version_imports fvi ON (fvi.feed_version_id = fv.id)
+        GROUP BY (fv.feed_id)
+      ) fvi_max
+      ON current_feeds.id = fvi_max.feed_id
+    })
+      .joins('INNER JOIN feed_version_imports ON (feed_version_imports.id = fvi_max.fvi_max_id)')
+      .where('feed_version_imports.success': import_status)
+    # Another approach, preserved here for now:
+    # see: http://stackoverflow.com/questions/121387/fetch-the-row-which-has-the-max-value-for-a-column/123481#123481
+    # LEFT OUTER JOIN feed_version_imports fvi2 ON (
+    #   fvi1.feed_version_id = fvi2.feed_version_id AND
+    #   fvi1.created_at < fvi2.created_at
+    # )
+    # WHERE fvi2.id IS NULL GROUP BY (fv.feed_id)
+  }
+
   include CurrentTrackedByChangeset
   current_tracked_by_changeset({
     kind_of_model_tracked: :onestop_entity,

spec/controllers/api/v1/feeds_controller_spec.rb

@@ -15,7 +15,7 @@
         })
       end
 
-      it 'filters by Onestop ID' do
+      it 'query: onestop_id' do
         create_list(:feed, 3)
         onestop_id = Feed.second.onestop_id
         get :index, onestop_id: onestop_id
@@ -56,7 +56,7 @@
         expect_json({feeds: -> (feeds) {expect(feeds.size).to eq(0)}})
       end
 
-      it 'filters on latest_fetch_exception' do
+      it 'query: latest_fetch_exception' do
         feeds = create_list(:feed, 3)
         feed = feeds.first
         feed.update!(latest_fetch_exception_log: 'test')
@@ -67,31 +67,53 @@
         get :index, latest_fetch_exception: 'false'
         expect_json({feeds: -> (feeds) {expect(feeds.size).to eq(2)}})
       end
+
+      it 'query: where_latest_feed_version_import_status' do
+        fvs = create_list(:feed_version, 3)
+        fvs[0].feed_version_imports.create!(success: true)
+        fvs[1].feed_version_imports.create!(success: false)
+        fvs[2].feed_version_imports.create!(success: nil)
+        get :index, latest_feed_version_import_status: 'true'
+        expect_json({feeds: -> (feeds) {
+          expect(feeds.length).to eq(1)
+          expect(feeds.first[:onestop_id]).to eq(fvs[0].feed.onestop_id)
+        }})
+        get :index, latest_feed_version_import_status: 'false'
+        expect_json({feeds: -> (feeds) {
+          expect(feeds.length).to eq(1)
+          expect(feeds.first[:onestop_id]).to eq(fvs[1].feed.onestop_id)
+        }})
+        get :index, latest_feed_version_import_status: 'null'
+        expect_json({feeds: -> (feeds) {
+          expect(feeds.length).to eq(1)
+          expect(feeds.first[:onestop_id]).to eq(fvs[2].feed.onestop_id)
+        }})
+      end
     end
-  end
 
-  context 'as GeoJSON' do
-    it 'should return GeoJSON for all feeds' do
-      create_list(:feed, 2, geometry: {
-        type: 'Polygon',
-        coordinates: [
-          [
-            [-71.04819, 42.254056],
-            [-70.9016389,42.254056],
-            [-70.9016389,42.359837],
-            [-71.04819,42.359837],
-            [-71.04819,42.254056]
+    context 'as GeoJSON' do
+      it 'should return GeoJSON for all feeds' do
+        create_list(:feed, 2, geometry: {
+          type: 'Polygon',
+          coordinates: [
+            [
+              [-71.04819, 42.254056],
+              [-70.9016389,42.254056],
+              [-70.9016389,42.359837],
+              [-71.04819,42.359837],
+              [-71.04819,42.254056]
+            ]
           ]
-        ]
-      })
+        })
 
-      get :index, format: :geojson
-      expect_json({
-        type: 'FeatureCollection',
-        features: -> (features) {
-          expect(features.map {|f| f[:properties][:onestop_id] }).to match_array(Feed.pluck(:onestop_id))
-        }
-      })
+        get :index, format: :geojson
+        expect_json({
+          type: 'FeatureCollection',
+          features: -> (features) {
+            expect(features.map {|f| f[:properties][:onestop_id] }).to match_array(Feed.pluck(:onestop_id))
+          }
+        })
+      end
     end
   end
 

spec/models/feed_spec.rb

@@ -363,6 +363,50 @@
     end
   end
 
+  context '.where_latest_feed_version_import_status' do
+    before(:each) do
+      # Create several feeds with different #'s of FVs and FVIs
+
+      # Last import: true
+      @feed1 = create(:feed)
+      @feed1_fv1 = create(:feed_version, feed: @feed1)
+      create(:feed_version_import, feed_version: @feed1_fv1, success: false, created_at: '2016-01-01')
+      create(:feed_version_import, feed_version: @feed1_fv1, success: true, created_at: '2016-01-02')
+
+      # Last import: false
+      @feed2 = create(:feed)
+      @feed2_fv1 = create(:feed_version, feed: @feed2)
+      create(:feed_version_import, feed_version: @feed2_fv1, success: true, created_at: '2016-01-01')
+      create(:feed_version_import, feed_version: @feed2_fv1, success: true, created_at: '2016-01-02')
+      @feed2_fv2 = create(:feed_version, feed: @feed2)
+      create(:feed_version_import, feed_version: @feed2_fv2, success: false, created_at: '2016-01-02')
+      create(:feed_version_import, feed_version: @feed2_fv2, success: false, created_at: '2016-01-03')
+      # create(:feed_version_import, feed_version: @feed2_fv2, success: false, created_at: '2016-01-03')
+
+      # Last import: nil
+      @feed3 = create(:feed)
+      @feed3_fv1 = create(:feed_version, feed: @feed3)
+      create(:feed_version_import, feed_version: @feed3_fv1, success: nil, created_at: '2016-01-01')
+
+      # Last import: does not exist
+      @feed4 = create(:feed)
+      @feed4_fv1 = create(:feed_version, feed: @feed4)
+    end
+
+    it 'finds successful import' do
+      expect(Feed.where_latest_feed_version_import_status(true)).to match_array([@feed1])
+    end
+
+    it 'finds failed import' do
+      expect(Feed.where_latest_feed_version_import_status(false)).to match_array([@feed2])
+    end
+
+    it 'finds in progress import' do
+      expect(Feed.where_latest_feed_version_import_status(nil)).to match_array([@feed3])
+    end
+
+  end
+
   context '.where_newer_feed_version' do
     before(:each) do
       date0 = Date.parse('2014-01-01')