Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
  • 4 commits
  • 6 files changed
  • 0 commit comments
  • 2 contributors
Commits on Dec 04, 2012
@jabley jabley Handle HMRC server being temperamental
Unable to download for some reason? Don't just loop until we hit VM
stack limits.
9b081d8
Commits on Dec 05, 2012
@jabley jabley Initial draft of archive functionality 1226605
Commits on Dec 11, 2012
@jabley jabley Add test for preventing infinite loop
Smug lisp weenies might wax about a decent condition system and
restarts...
a943147
Commits on Dec 12, 2012
@saulius saulius Merge pull request #40 from alphagov/snapshot-updates
Snapshot updates
e96c539
View
13 lib/tariff_synchronizer.rb
@@ -37,6 +37,9 @@ module TariffSynchronizer
autoload :PendingUpdate, 'tariff_synchronizer/pending_update'
autoload :TaricUpdate, 'tariff_synchronizer/taric_update'
autoload :ChiefUpdate, 'tariff_synchronizer/chief_update'
+ autoload :TaricArchive, "tariff_synchronizer/taric_archive"
+ autoload :ChiefArchive, "tariff_synchronizer/chief_archive"
+
extend self
@@ -92,6 +95,16 @@ def download
end
end
+ def download_archive
+ if sync_variables_set?
+ ActiveSupport::Notifications.instrument("download.tariff_synchronizer") do
+ [TaricArchive, ChiefArchive].map(&:sync)
+ end
+ else
+ ActiveSupport::Notifications.instrument("config_error.tariff_synchronizer")
+ end
+ end
+
# Applies all updates (from inbox/failbox) by their date starting from the
# oldest one.
def apply
View
35 lib/tariff_synchronizer/chief_archive.rb
@@ -0,0 +1,35 @@
+require 'tariff_synchronizer/chief_update'
+require 'tariff_synchronizer/file_service'
+
+module TariffSynchronizer
+ class ChiefArchive < ChiefUpdate
+ class << self
+ def download(date)
+ begin
+ super
+ rescue
+ ActiveSupport::Notifications.instrument("not_found.tariff_synchronizer", date: date,
+ url: chief_update_url_for(date))
+ end
+ end
+
+ def exists_for?(date)
+ # We can check the filesystem to see if we have a file locally (that is zero-length?)
+ local_file = update_path(date, file_name_for(date))
+ File.exists?(local_file) && File.stat(local_file).size != 0
+ end
+
+ def pending_from
+ if ENV["CHIEF_START_DATE"].present?
+ Date.strptime(ENV["CHIEF_START_DATE"], "%Y-%m-%d")
+ else
+ TariffSynchronizer.initial_update_for(update_type)
+ end
+ end
+
+ def create_update_entry(date, state, file_name = file_name_for(date))
+ # no-op
+ end
+ end
+ end
+end
View
8 lib/tariff_synchronizer/file_service.rb
@@ -45,7 +45,7 @@ def download_content(url)
private
- def send_request(url)
+ def send_request(url, count = 0)
begin
crawler = Curl::Easy.new(url)
crawler.ssl_verify_peer = false
@@ -57,7 +57,11 @@ def send_request(url)
rescue Curl::Err::HostResolutionError => exception
# NOTE could be a glitch in curb because it throws HostResolutionError
# occasionally without any reason.
- send_request(url)
+ if count < 10
+ send_request(url, count + 1)
+ else
+ raise StandardError.new(exception.to_s + " for <#{url}>")
+ end
end
return Response.new(url, crawler.response_code, crawler.body_str)
View
36 lib/tariff_synchronizer/taric_archive.rb
@@ -0,0 +1,36 @@
+require 'tariff_synchronizer/taric_update'
+require 'tariff_synchronizer/file_service'
+
+module TariffSynchronizer
+ class TaricArchive < TaricUpdate
+ class << self
+ def download(date)
+ begin
+ super
+ rescue
+ ActiveSupport::Notifications.instrument("not_found.tariff_synchronizer", date: date,
+ url: taric_query_url_for(date))
+ end
+ end
+
+ def exists_for?(date)
+ # Not sure we can check for existing content - sadpanda being a bad HTTP citizen.
+ # Reason being, TARIC can have multiple update files for a given date; we can't
+ # predict the existence of the file.
+ false
+ end
+
+ def pending_from
+ if ENV["TARIC_START_DATE"].present?
+ Date.strptime(ENV["TARIC_START_DATE"], "%Y-%m-%d")
+ else
+ TariffSynchronizer.initial_update_for(update_type)
+ end
+ end
+
+ def create_update_entry(date, state, file_name = file_name_for(date))
+ # no-op
+ end
+ end
+ end
+end
View
5 lib/tasks/tariff.rake
@@ -41,6 +41,11 @@ namespace :tariff do
end
end
+ desc "Download all Taric and CHIEF updates"
+ task download: :environment do
+ TariffSynchronizer.download_archive
+ end
+
desc 'Apply pending Taric and CHIEF'
task transform: %w[environment] do
require 'chief_transformer'
View
9 spec/unit/tariff_synchronizer/file_service_spec.rb
@@ -62,5 +62,14 @@
subject.download_content(example_url).should == error_response
end
+
+ it "will abort send_request if it gets too many HostResolutionErrors" do
+ # Very promiscuous knowledge of the internals here!
+ Curl::Easy.any_instance.stubs(:perform).raises(Curl::Err::HostResolutionError)
+
+ expect {
+ subject.download_content(example_url)
+ }.to raise_error(StandardError)
+ end
end
end

No commit comments for this range

Something went wrong with that request. Please try again.