Tariff moved to Point in Time DB architecture.

This adds oplog tables for TARIC tables and reconstructs current state with the help of DB views. It enables us to rollback to any point time backwards. This enables TARIC validations that we have currently in place. The validation framework was rewritten to have reasonable record error messages. TARIC importer got upgrade in relation with enabled validations. It also uses less memory due to reduced usage of Nokogiri (uses hashes for record parsing now).
alphagov · Apr 15, 2013 · 7840e33 · 7840e33
1 parent 62c5e1d
commit 7840e33
Show file tree

Hide file tree

Showing 251 changed files with 26,884 additions and 24,340 deletions.
diff --git a/Gemfile b/Gemfile
@@ -4,13 +4,14 @@ source "https://BnrJb6FZyzspBboNJzYZ@gem.fury.io/govuk/"
 gem "rails", "3.2.13"
 
 gem "addressable", "2.3.2"
-gem "mysql2"
 gem "multi_json"
-gem "yajl-ruby", require: "yajl"
+gem "mysql2"
+gem "nulogy-sequel-rails", "0.3.9"
 gem "rabl", "0.7.6"
+gem "ruby-progressbar"
 gem "sequel"
-gem "nulogy-sequel-rails", "0.3.9"
 gem "tire"
+gem "yajl-ruby", require: "yajl"
 
 gem "builder"
 gem "jquery-rails", "1.0.19"

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -174,6 +174,7 @@ GEM
       rspec-core (~> 2.13.0)
       rspec-expectations (~> 2.13.0)
       rspec-mocks (~> 2.13.0)
+    ruby-progressbar (1.0.2)
     ruby2ruby (1.3.1)
       ruby_parser (~> 2.0)
       sexp_processor (~> 3.0)
@@ -263,6 +264,7 @@ DEPENDENCIES
   railties
   router-client (= 3.0.1)
   rspec-rails
+  ruby-progressbar
   sass-rails
   sequel
   shoulda-matchers (>= 1.5.1)

diff --git a/app/controllers/api/v1/commodities_controller.rb b/app/controllers/api/v1/commodities_controller.rb
@@ -7,7 +7,7 @@ def show
         @measures = MeasurePresenter.new(@commodity.measures_dataset.eager({geographical_area: [:geographical_area_descriptions,
                                                                                                 { contained_geographical_areas: :geographical_area_descriptions }]},
                                                       {footnotes: :footnote_descriptions},
-                                                      {type: :measure_type_description},
+                                                      {measure_type: :measure_type_description},
                                                       {measure_components: [{duty_expression: :duty_expression_description},
                                                                             {measurement_unit: :measurement_unit_description},
                                                                             :monetary_unit,

diff --git a/app/controllers/api/v1/headings_controller.rb b/app/controllers/api/v1/headings_controller.rb
@@ -10,7 +10,7 @@ def show
           @measures = MeasurePresenter.new(@heading.measures_dataset.eager({geographical_area: [:geographical_area_descriptions,
                                                                                                 { contained_geographical_areas: :geographical_area_descriptions }]},
                                                       {footnotes: :footnote_descriptions},
-                                                      {type: :measure_type_description},
+                                                      {measure_type: :measure_type_description},
                                                       {measure_components: [{duty_expression: :duty_expression_description},
                                                                             {measurement_unit: :measurement_unit_description},
                                                                             :monetary_unit,

diff --git a/app/models/additional_code.rb b/app/models/additional_code.rb
@@ -1,5 +1,6 @@
 class AdditionalCode < Sequel::Model
   plugin :time_machine
+  plugin :oplog, primary_key: :additional_code_sid
 
   set_primary_key :additional_code_sid
 
@@ -22,44 +23,9 @@ def additional_code_description
 
   delegate :description, to: :additional_code_description
 
-  def exists_as_meursing_code?
-    meursing_additional_code.present?
-  end
-
-  def does_not_exist_as_meursing_code?
-    !exists_as_meursing_code?
-  end
-
-  def exists_as_export_refund_code?
-    export_refund_nomenclature.present?
-  end
-
   def code
     "#{additional_code_type_id}#{additional_code}"
   end
-
-  ######### Conformance validations 245
-  validates do
-    # ACN1
-    # TODO: ACN2
-    uniqueness_of [:additional_code, :additional_code_type_id, :validity_start_date]
-    # ACN3
-    validity_dates
-    # TODO: ACN4
-    # TODO: ACN12
-    # TODO: ACN13
-    # TODO: ACN17
-    # TODO: ACN6
-    # TODO: ACN7
-    # TODO: ACN8
-    # TODO: ACN9
-    # TODO: ACN10
-    # TODO: ACN11
-    # TODO: ACN16
-    # TODO: ACN5
-    # TODO: ACN14
-    # TODO: ACN15
-  end
 end
 
 
diff --git a/app/models/additional_code_description.rb b/app/models/additional_code_description.rb
@@ -1,5 +1,6 @@
 class AdditionalCodeDescription < Sequel::Model
   plugin :time_machine
+  plugin :oplog, primary_key: [:additional_code_description_period_sid, :additional_code_sid]
 
   set_primary_key [:additional_code_description_period_sid, :additional_code_sid]
 

diff --git a/app/models/additional_code_description_period.rb b/app/models/additional_code_description_period.rb
@@ -1,4 +1,7 @@
 class AdditionalCodeDescriptionPeriod < Sequel::Model
+  plugin :oplog, primary_key: [:additional_code_description_period_sid,
+                   :additional_code_sid,
+                   :additional_code_type_id]
   plugin :time_machine
 
   set_primary_key [:additional_code_description_period_sid,

diff --git a/app/models/additional_code_type.rb b/app/models/additional_code_type.rb
@@ -1,4 +1,6 @@
 class AdditionalCodeType < Sequel::Model
+  plugin :oplog, primary_key: :additional_code_type_id
+
   set_primary_key :additional_code_type_id
 
   one_to_many :additional_codes, key: :additional_code_type_id
@@ -24,42 +26,6 @@ class AdditionalCodeType < Sequel::Model
     4 => "Export refund for processed agricultural goods"
   }
 
-  ######### Conformance validations 120
-  validates do
-    # CT1
-    uniqueness_of :additional_code_type_id
-    # CT2
-    input_of :meursing_table_plan_id, if: :meursing?
-    # CT3
-    associated :meursing_table_plan, ensure: :meursing_table_plan_present?,
-                                     if: :should_validate_meursing_table_plan?
-    # CT4
-    validity_dates
-  end
-
-  def before_destroy
-    # CT6
-    return false if additional_codes.select{|adco| adco.meursing_additional_code.blank? }.any?
-    # CT7
-    return false if meursing_table_plan_id.present?
-    # CT9
-    return false if export_refund? && export_refund_nomenclatures.any?
-    # CT10
-    return false if measure_types.any?
-    # TODO CT11
-    # return false if
-
-    super
-  end
-
-  def should_validate_meursing_table_plan?
-    meursing_table_plan_id.present? && meursing?
-  end
-
-  def related_to_measure_type?
-    measure_types.any?
-  end
-
   def meursing?
     application_code == "3"
   end
@@ -76,5 +42,3 @@ def export_refund_agricultural?
     application_code == "4"
   end
 end
-
-
diff --git a/app/models/additional_code_type_description.rb b/app/models/additional_code_type_description.rb
@@ -1,4 +1,6 @@
 class AdditionalCodeTypeDescription < Sequel::Model
+  plugin :oplog, primary_key: :additional_code_type_id
+
   set_primary_key :additional_code_type_id
 
   many_to_one :additional_code_type, key: :additional_code_type_id

diff --git a/app/models/additional_code_type_measure_type.rb b/app/models/additional_code_type_measure_type.rb
@@ -1,20 +1,8 @@
 class AdditionalCodeTypeMeasureType < Sequel::Model
+  plugin :oplog, primary_key: [:measure_type_id, :additional_code_type_id]
+
   set_primary_key [:measure_type_id, :additional_code_type_id]
 
   many_to_one :measure_type
   many_to_one :additional_code_type
-
-  ######### Conformance validations 240
-  validates do
-    # AMT1, AMT2
-    presence_of [:measure_type_id, :additional_code_type_id]
-    # AMT3
-    uniqueness_of [:measure_type_id, :additional_code_type_id]
-    # AMT5
-    validity_dates
-    # TODO: AMT4
-    # TODO: AMT7
-  end
 end
-
-
diff --git a/app/models/base_regulation.rb b/app/models/base_regulation.rb
@@ -1,4 +1,5 @@
 class BaseRegulation < Sequel::Model
+  plugin :oplog, primary_key: [:base_regulation_id, :base_regulation_role]
   plugin :time_machine, period_start_column: :base_regulations__validity_start_date,
                         period_end_column: :effective_end_date
 
@@ -11,65 +12,12 @@ def not_completely_abrogated?
     complete_abrogation_regulation.blank?
   end
 
-  ######### Conformance validations 285
-  validates do
-    # ROIMB1
-    uniqueness_of [:base_regulation_id, :base_regulation_role]
-    # ROIMB3
-    validity_dates
-    # TODO: ROIMB4
-    # TODO: ROIMB5
-    # TODO: ROIMB6
-    # TODO: ROIMB7
-    # TODO: ROIMB48
-    # TODO: ROIMB44
-    # TODO: ROIMB46
-    # TODO: ROIMB47
-    # TODO: ROIMB8
-    # TODO: ROIMB9
-    # TODO: ROIMB10
-    # TODO: ROIMB11
-    # TODO: ROIMB12
-    # TODO: ROIMB13
-    # TODO: ROIMB14
-    # TODO: ROIMB15
-    # TODO: ROIMB16
-    # TODO: ROIMB17
-    # TODO: ROIMB18
-    # TODO: ROIMB19
-    # TODO: ROIMB20
-    # TODO: ROIMB21
-    # TODO: ROIMB22
-    # TODO: ROIMB23
-    # TODO: ROIMB24
-    # TODO: ROIMB25
-    # TODO: ROIMB26
-    # TODO: ROIMB27
-    # TODO: ROIMB28
-    # TODO: ROIMB29
-    # TODO: ROIMB30
-    # TODO: ROIMB31
-    # TODO: ROIMB32
-    # TODO: ROIMB33
-    # TODO: ROIMB34
-    # TODO: ROIMB35
-    # TODO: ROIMB36
-    # TODO: ROIMB37
-    # TODO: ROIMB38
-    # TODO: ROIMB39
-    # TODO: ROIMB40
-    # TODO: ROIMB41
-    # TODO: ROIMB43
-    # TODO: ROAC4
-    # TODO: ROAC6
-    # TODO: ROAC7
-    # TODO: ROAC8
-    # TODO: ROAC10
-    # TODO: ROAE4
-    # TODO: ROAE6
-    # TODO: ROAE7
-    # TODO: ROAE8
-    # TODO: ROAE11
+  # TODO confirm this assumption
+  # 0 not replaced
+  # 1 fully replaced
+  # 2 partially replaced
+  def fully_replaced?
+    replacement_indicator == 1
   end
 end
 

diff --git a/app/models/certificate.rb b/app/models/certificate.rb
@@ -1,4 +1,5 @@
 class Certificate < Sequel::Model
+  plugin :oplog, primary_key: [:certificate_code, :certificate_type_code]
   plugin :time_machine
 
   set_primary_key [:certificate_code, :certificate_type_code]

diff --git a/app/models/certificate_description.rb b/app/models/certificate_description.rb
@@ -1,4 +1,5 @@
 class CertificateDescription < Sequel::Model
+  plugin :oplog, primary_key: [:certificate_description_period_sid]
   plugin :time_machine
 
   set_primary_key :certificate_description_period_sid

diff --git a/app/models/certificate_description_period.rb b/app/models/certificate_description_period.rb
@@ -1,4 +1,5 @@
 class CertificateDescriptionPeriod < Sequel::Model
+  plugin :oplog, primary_key: [:certificate_description_period_sid]
   plugin :time_machine
 
   set_primary_key :certificate_description_period_sid

diff --git a/app/models/certificate_type.rb b/app/models/certificate_type.rb
@@ -1,4 +1,5 @@
 class CertificateType < Sequel::Model
+  plugin :oplog, primary_key: :certificate_type_code
   plugin :time_machine
 
   set_primary_key :certificate_type_code
@@ -11,21 +12,6 @@ class CertificateType < Sequel::Model
                              primary_key: :certificate_type_code
 
   delegate :description, to: :certificate_type_description
-
-  ######### Conformance validations 110
-  validates do
-    # CET1
-    uniqueness_of :certificate_type_code
-    # CET3
-    validity_dates
-  end
-
-  def before_destroy
-    # CET2
-    return false if certificates.any?
-
-    super
-  end
 end
 
 
diff --git a/app/models/certificate_type_description.rb b/app/models/certificate_type_description.rb
@@ -1,4 +1,5 @@
 class CertificateTypeDescription < Sequel::Model
+  plugin :oplog, primary_key: :certificate_type_code
   set_primary_key  :certificate_type_code
 end
 

diff --git a/app/models/chapter.rb b/app/models/chapter.rb
@@ -4,6 +4,7 @@ class Chapter < GoodsNomenclature
   include Tire::Model::Search
 
   plugin :json_serializer
+  plugin :oplog, primary_key: :goods_nomenclature_sid
 
   set_dataset filter("goods_nomenclatures.goods_nomenclature_item_id LIKE ?", '__00000000').
               order(:goods_nomenclature_item_id.asc)

diff --git a/app/models/chief/mfcm.rb b/app/models/chief/mfcm.rb
@@ -46,6 +46,12 @@ class Mfcm < Sequel::Model
                                                  chief_measure_type_adco__tax_type_code: tty_code) },
                                                  class_name: 'Chief::MeasureTypeAdco'
 
+    one_to_one :chief_update, key: :filename,
+                              primary_key: :origin,
+                              class_name: TariffSynchronizer::ChiefUpdate
+
+    delegate :issue_date, to: :chief_update, allow_nil: true
+
     dataset_module do
       def unprocessed
         filter(processed: false)
@@ -78,5 +84,9 @@ def measure_type_adco
     def audit_tsmp
       self[:audit_tsmp].presence || Time.now
     end
+
+    def operation_date
+      issue_date
+    end
   end
 end
diff --git a/app/models/chief/tame.rb b/app/models/chief/tame.rb
@@ -39,6 +39,12 @@ class Tame < Sequel::Model
                         }.order(:audit_tsmp.asc)
     }
 
+    one_to_one :chief_update, key: :filename,
+                              primary_key: :origin,
+                              class_name: TariffSynchronizer::ChiefUpdate
+
+    delegate :issue_date, to: :chief_update, allow_nil: true
+
     dataset_module do
       def unprocessed
         filter(processed: false)
@@ -63,6 +69,10 @@ def has_tamfs?
     def audit_tsmp
       self[:audit_tsmp].presence || Time.now
     end
+
+    def operation_date
+      issue_date
+    end
   end
 end