Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
  • 4 commits
  • 6 files changed
  • 0 commit comments
  • 2 contributors
Commits on Dec 04, 2012
James Abley jabley Handle HMRC server being temperamental
Unable to download for some reason? Don't just loop until we hit VM
stack limits.
9b081d8
Commits on Dec 05, 2012
James Abley jabley Initial draft of archive functionality 1226605
Commits on Dec 11, 2012
James Abley jabley Add test for preventing infinite loop
Smug lisp weenies might wax about a decent condition system and
restarts...
a943147
Commits on Dec 12, 2012
Saulius Grigaliunas saulius Merge pull request #40 from alphagov/snapshot-updates
Snapshot updates
e96c539
13 lib/tariff_synchronizer.rb
View
@@ -37,6 +37,9 @@ module TariffSynchronizer
autoload :PendingUpdate, 'tariff_synchronizer/pending_update'
autoload :TaricUpdate, 'tariff_synchronizer/taric_update'
autoload :ChiefUpdate, 'tariff_synchronizer/chief_update'
+ autoload :TaricArchive, "tariff_synchronizer/taric_archive"
+ autoload :ChiefArchive, "tariff_synchronizer/chief_archive"
+
extend self
@@ -92,6 +95,16 @@ def download
end
end
+ def download_archive
+ if sync_variables_set?
+ ActiveSupport::Notifications.instrument("download.tariff_synchronizer") do
+ [TaricArchive, ChiefArchive].map(&:sync)
+ end
+ else
+ ActiveSupport::Notifications.instrument("config_error.tariff_synchronizer")
+ end
+ end
+
# Applies all updates (from inbox/failbox) by their date starting from the
# oldest one.
def apply
35 lib/tariff_synchronizer/chief_archive.rb
View
@@ -0,0 +1,35 @@
+require 'tariff_synchronizer/chief_update'
+require 'tariff_synchronizer/file_service'
+
+module TariffSynchronizer
+ class ChiefArchive < ChiefUpdate
+ class << self
+ def download(date)
+ begin
+ super
+ rescue
+ ActiveSupport::Notifications.instrument("not_found.tariff_synchronizer", date: date,
+ url: chief_update_url_for(date))
+ end
+ end
+
+ def exists_for?(date)
+ # We can check the filesystem to see if we have a file locally (that is zero-length?)
+ local_file = update_path(date, file_name_for(date))
+ File.exists?(local_file) && File.stat(local_file).size != 0
+ end
+
+ def pending_from
+ if ENV["CHIEF_START_DATE"].present?
+ Date.strptime(ENV["CHIEF_START_DATE"], "%Y-%m-%d")
+ else
+ TariffSynchronizer.initial_update_for(update_type)
+ end
+ end
+
+ def create_update_entry(date, state, file_name = file_name_for(date))
+ # no-op
+ end
+ end
+ end
+end
8 lib/tariff_synchronizer/file_service.rb
View
@@ -45,7 +45,7 @@ def download_content(url)
private
- def send_request(url)
+ def send_request(url, count = 0)
begin
crawler = Curl::Easy.new(url)
crawler.ssl_verify_peer = false
@@ -57,7 +57,11 @@ def send_request(url)
rescue Curl::Err::HostResolutionError => exception
# NOTE could be a glitch in curb because it throws HostResolutionError
# occasionally without any reason.
- send_request(url)
+ if count < 10
+ send_request(url, count + 1)
+ else
+ raise StandardError.new(exception.to_s + " for <#{url}>")
+ end
end
return Response.new(url, crawler.response_code, crawler.body_str)
36 lib/tariff_synchronizer/taric_archive.rb
View
@@ -0,0 +1,36 @@
+require 'tariff_synchronizer/taric_update'
+require 'tariff_synchronizer/file_service'
+
+module TariffSynchronizer
+ class TaricArchive < TaricUpdate
+ class << self
+ def download(date)
+ begin
+ super
+ rescue
+ ActiveSupport::Notifications.instrument("not_found.tariff_synchronizer", date: date,
+ url: taric_query_url_for(date))
+ end
+ end
+
+ def exists_for?(date)
+ # Not sure we can check for existing content - sadpanda being a bad HTTP citizen.
+ # Reason being, TARIC can have multiple update files for a given date; we can't
+ # predict the existence of the file.
+ false
+ end
+
+ def pending_from
+ if ENV["TARIC_START_DATE"].present?
+ Date.strptime(ENV["TARIC_START_DATE"], "%Y-%m-%d")
+ else
+ TariffSynchronizer.initial_update_for(update_type)
+ end
+ end
+
+ def create_update_entry(date, state, file_name = file_name_for(date))
+ # no-op
+ end
+ end
+ end
+end
5 lib/tasks/tariff.rake
View
@@ -41,6 +41,11 @@ namespace :tariff do
end
end
+ desc "Download all Taric and CHIEF updates"
+ task download: :environment do
+ TariffSynchronizer.download_archive
+ end
+
desc 'Apply pending Taric and CHIEF'
task transform: %w[environment] do
require 'chief_transformer'
9 spec/unit/tariff_synchronizer/file_service_spec.rb
View
@@ -62,5 +62,14 @@
subject.download_content(example_url).should == error_response
end
+
+ it "will abort send_request if it gets too many HostResolutionErrors" do
+ # Very promiscuous knowledge of the internals here!
+ Curl::Easy.any_instance.stubs(:perform).raises(Curl::Err::HostResolutionError)
+
+ expect {
+ subject.download_content(example_url)
+ }.to raise_error(StandardError)
+ end
end
end

No commit comments for this range

Something went wrong with that request. Please try again.