Perform Delayed Sync Job up to 1 additional time

wcc-contentful-app/app/helpers/wcc/contentful/app/section_helper.rb

@@ -30,16 +30,16 @@ def section_id(section)
 
   def markdown(text)
     options = {
-      filter_html:     true,
-      hard_wrap:       true,
+      filter_html: true,
+      hard_wrap: true,
       link_attributes: { rel: 'nofollow', target: '_blank' },
       space_after_headers: true,
       fenced_code_blocks: true
     }
 
     extensions = {
-      autolink:           true,
-      superscript:        true,
+      autolink: true,
+      superscript: true,
       disable_indented_code_blocks: true,
       tables: true
     }

wcc-contentful/app/jobs/wcc/contentful/delayed_sync_job.rb

@@ -18,7 +18,11 @@ def self.mutex
     def perform(event = nil)
       up_to_id = nil
       up_to_id = event[:up_to_id] || event.dig('sys', 'id') if event
-      sync!(up_to_id: up_to_id)
+      # Re-enqueue a second time if there is no event[:up_to_id]
+      # If there is an event[:up_to_id], that means we have previously enqueued
+      go_again = false
+      go_again = !event[:up_to_id] if event
+      sync!(up_to_id: up_to_id, go_again: go_again)
     end
 
     # Calls the Contentful Sync API and updates the configured store with the returned
@@ -29,7 +33,7 @@ def perform(event = nil)
     #  If we don't find this ID in the sync data, then drop a job to try
     #  the sync again after a few minutes.
     #
-    def sync!(up_to_id: nil)
+    def sync!(up_to_id: nil, go_again: false)
       return unless store.respond_to?(:index)
 
       self.class.mutex.synchronize do
@@ -48,7 +52,8 @@ def sync!(up_to_id: nil)
         store.set('sync:token', token: sync_resp.next_sync_token)
 
         logger.info "Synced #{count} entries.  Next sync token:\n  #{sync_resp.next_sync_token}"
-        sync_later!(up_to_id: up_to_id) unless id_found
+        logger.info "Should enqueue again? [#{!id_found && go_again}]"
+        sync_later!(up_to_id: up_to_id) if !id_found && go_again
         sync_resp.next_sync_token
       end
     end

wcc-contentful/lib/wcc/contentful/client_ext.rb

@@ -19,9 +19,9 @@ def self.get_http(url, query, headers = {}, proxy = {})
     adapter.call(url, query, headers, proxy)
   end
 
-  REWRITE_REGEXP = /^(https?\:\/\/(?:\w+)\.contentful\.com\/spaces\/[^\/]+\/)(?!environments)(.+)$/
+  REWRITE_REGX = /^(https?\:\/\/(?:\w+)\.contentful\.com\/spaces\/[^\/]+\/)(?!environments)(.+)$/
   def self.rewrite_to_environment(url, environment)
-    return url unless m = REWRITE_REGEXP.match(url)
+    return url unless m = REWRITE_REGX.match(url)
 
     File.join(m[1], 'environments', environment, m[2])
   end

wcc-contentful/spec/jobs/wcc/contentful/delayed_sync_job_spec.rb

@@ -80,7 +80,7 @@
       end
     end
 
-    it 'Drops the job again if the ID still does not come back' do
+    it 'Drops the job again if the ID still does not come back and told to go again' do
       allow(client).to receive(:sync)
         .and_return(double(
                       items: next_sync['items'],
@@ -92,15 +92,30 @@
         .with(up_to_id: 'foobar')
 
       # act
-      job.sync!(up_to_id: 'foobar')
+      job.sync!(up_to_id: 'foobar', go_again: true)
+    end
+
+    it 'Drops the job again if the ID still does not come back and told not to go again' do
+      allow(client).to receive(:sync)
+        .and_return(double(
+                      items: next_sync['items'],
+                      next_sync_token: 'test2'
+                    ))
+
+      # expect
+      expect(job).to_not receive(:sync_later!)
+        .with(up_to_id: 'foobar')
+
+      # act
+      job.sync!(up_to_id: 'foobar', go_again: false)
     end
   end
 
   describe 'Perform' do
     it 'calls into job.sync!' do
       expect_any_instance_of(described_class)
         .to receive(:sync!)
-        .with(up_to_id: nil)
+        .with(up_to_id: nil, go_again: false)
 
       # act
       described_class.perform_now
@@ -109,7 +124,7 @@
     it 'calls into job.sync! with explicit params' do
       expect_any_instance_of(described_class)
         .to receive(:sync!)
-        .with(up_to_id: 'asdf')
+        .with(up_to_id: 'asdf', go_again: false)
 
       # act
       described_class.perform_now(up_to_id: 'asdf')
@@ -118,7 +133,7 @@
     it 'calls into job.sync! with webhook event' do
       expect_any_instance_of(described_class)
         .to receive(:sync!)
-        .with(up_to_id: 'testId1')
+        .with(up_to_id: 'testId1', go_again: true)
 
       # act
       described_class.perform_now({

wcc-contentful/spec/spec_helper.rb

@@ -8,6 +8,7 @@
 require 'webmock/rspec'
 require 'vcr'
 require 'httplog'
+require 'byebug'
 
 require 'bench_helper'
 

wcc-contentful/wcc-contentful.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
 
   spec.require_paths = ['lib']
 
+  spec.add_development_dependency 'byebug'
   spec.add_development_dependency 'coveralls'
   spec.add_development_dependency 'dotenv', '~> 2.2'
   spec.add_development_dependency 'erb_lint', '~> 0.0.26'