Add correlation_id to models and make public_id auto incrementing.

app/models/activity.rb

@@ -2,11 +2,15 @@ class Activity
   include Mongoid::Document
   field :public_id, :type => Integer
   index :public_id, :unique => true
+  field :correlation_id, :type => Integer
+  index :correlation_id, :unique => true
   field :name, :type => String
   has_and_belongs_to_many :sectors
 
   validates :name, :presence => true
 
+  before_save :set_public_id
+
   def self.find_by_public_id(public_id)
     where(public_id: public_id).first
   end
@@ -15,6 +19,10 @@ def self.find_by_public_ids(public_ids)
     self.any_in public_id: public_ids
   end
 
+  def self.find_by_correlation_id(correlation_id)
+    where(correlation_id: correlation_id).first
+  end
+
   def self.find_by_sectors(sectors)
     activity_ids = sectors.map(&:activity_ids).flatten
     self.any_in _id: activity_ids
@@ -23,4 +31,17 @@ def self.find_by_sectors(sectors)
   def to_s
     self.name
   end
+
+  def set_public_id
+    # TODO: factor out
+    if self.public_id.nil?
+      counter = Mongoid.database["counters"].find_and_modify(
+        :query  => {:_id  => "activity"},
+        :update => {:$inc => {:count => 1}},
+        :new    => true,
+        :upsert => true
+      )["count"]
+      self.public_id = counter
+    end
+  end
 end

app/models/licence.rb

@@ -2,6 +2,8 @@ class Licence
   include Mongoid::Document
   field :public_id, :type => Integer
   index :public_id, :unique => true
+  field :correlation_id, :type => Integer
+  index :correlation_id, :unique => true
   field :name, :type => String
   field :regulation_area, :type => String
   field :da_england, :type => Boolean
@@ -12,13 +14,32 @@ class Licence
   validates :name, :presence => true
   validates :regulation_area, :presence => true
 
+  before_save :set_public_id
+
   def self.find_by_public_id(public_id)
     where(public_id: public_id).first
   end
 
+  def self.find_by_correlation_id(correlation_id)
+    where(correlation_id: correlation_id).first
+  end
+
   def self.find_by_sectors_activities_and_location(sectors, activities, location)
     # consider moving location into licence_link to avoid two queries
     licence_links = LicenceLink.find_by_sectors_and_activities(sectors, activities)
     where(:_id.in => licence_links.map(&:licence_id), "da_#{location}".to_sym => true)
   end
+
+  def set_public_id
+    # TODO: factor out
+    if self.public_id.nil?
+      counter = Mongoid.database["counters"].find_and_modify(
+          :query  => {:_id  => "licence"},
+          :update => {:$inc => {:count => 1}},
+          :new    => true,
+          :upsert => true
+      )["count"]
+      self.public_id = counter
+    end
+  end
 end

app/models/sector.rb

@@ -2,13 +2,17 @@ class Sector
   include Mongoid::Document
   field :public_id, :type => Integer
   index :public_id, :unique => true
+  field :correlation_id, :type => Integer
+  index :correlation_id, :unique => true
   field :name, :type => String
   field :layer1_id, :type => Integer # Only for the purpose of importing
   field :layer2_id, :type => Integer # Only for the purpose of importing
   has_and_belongs_to_many :activities
 
   validates :name, :presence => true
 
+  before_save :set_public_id
+
   def self.find_by_public_id(public_id)
     where(public_id: public_id).first
   end
@@ -17,6 +21,23 @@ def self.find_by_public_ids(public_ids)
     self.any_in public_id: public_ids
   end
 
+  def self.find_by_correlation_id(correlation_id)
+    where(correlation_id: correlation_id).first
+  end
+
+  def set_public_id
+    # TODO: factor out
+    if self.public_id.nil?
+      counter = Mongoid.database["counters"].find_and_modify(
+        :query  => {:_id => "sector"},
+        :update => {:$inc => {:count => 1}},
+        :new    => true,
+        :upsert => true
+      )['count']
+      self.public_id = counter
+    end
+  end
+
   def to_s
     self.name
   end

lib/data_importer/activities.rb

@@ -8,13 +8,13 @@ def self.open_data_file
   private
 
   def process_row(row)
-    unless sector = Sector.find_by_public_id(row['SECTOR_OID'].to_i)
+    unless sector = Sector.find_by_correlation_id(row['SECTOR_OID'].to_i)
       Rails.logger.info "Can't find Sector #{row['SECTOR_OID']} (#{row['SECTOR']})"
       return
     end
-    unless activity = Activity.find_by_public_id(row['BUSINSS_ACT_ID'].to_i)
+    unless activity = Activity.find_by_correlation_id(row['BUSINSS_ACT_ID'].to_i)
       activity = Activity.new
-      activity.public_id = row['BUSINSS_ACT_ID'].to_i
+      activity.correlation_id = row['BUSINSS_ACT_ID'].to_i
       activity.name = row['ACTIVITY_TITLE']
       Rails.logger.debug "Creating BusinessActivity #{activity.id}(#{activity.name})"
       activity.safely.save!

lib/data_importer/licences.rb

@@ -12,12 +12,12 @@ def process_row(row)
     if sectors.length == 0
       raise "Could not find sector #{row['SECTOR_OID']}, failing."
     end
-    unless activity = Activity.find_by_public_id(row['BUSINESSACT_ID'].to_i)
+    unless activity = Activity.find_by_correlation_id(row['BUSINESSACT_ID'].to_i)
       raise "Could not find activity #{row['BUSINESSACT_ID']}, failing."
     end
-    unless licence = Licence.find_by_public_id(row['LICENCE_OID'].to_i)
+    unless licence = Licence.find_by_correlation_id(row['LICENCE_OID'].to_i)
       licence = Licence.new
-      licence.public_id = row['LICENCE_OID'].to_i
+      licence.correlation_id = row['LICENCE_OID'].to_i
       Rails.logger.debug "Creating licence #{licence.id}(#{licence.name})"
     end
     licence.name = row['LICENCE']
@@ -46,7 +46,7 @@ def find_licence_link(sector, activity, licence)
   end
 
   def find_sectors(sector_id)
-    if sector = Sector.find_by_public_id(sector_id)
+    if sector = Sector.find_by_correlation_id(sector_id)
       [sector]
     else
       Sector.any_of({layer1_id: sector_id}, {layer2_id: sector_id})

lib/data_importer/sectors.rb

@@ -8,11 +8,11 @@ def self.open_data_file
   private
 
   def process_row(row)
-    if Sector.find_by_public_id(row['LAYER3_OID'].to_i)
+    if Sector.find_by_correlation_id(row['LAYER3_OID'].to_i)
       Rails.logger.info "Skipping Layer3 sector #{row['LAYER3_OID']}(#{row['LAYER3']})"
     else
       layer3 = Sector.new
-      layer3.public_id = row['LAYER3_OID'].to_i
+      layer3.correlation_id = row['LAYER3_OID'].to_i
       layer3.name = row['LAYER3']
       layer3.layer1_id = row['LAYER1_OID'].to_i
       layer3.layer2_id = row['LAYER2_OID'].to_i

spec/factories/activities.rb

@@ -3,6 +3,7 @@
 FactoryGirl.define do
   factory :activity do
     sequence(:public_id)
+    sequence(:correlation_id)
     name "Test Activity"
   end
 end

spec/factories/licences.rb

@@ -3,6 +3,7 @@
 FactoryGirl.define do
   factory :licence do
     sequence(:public_id)
+    sequence(:correlation_id)
     name "Test Licence"
     regulation_area "Test Regulation Area"
     da_england true

spec/factories/sectors.rb

@@ -3,6 +3,7 @@
 FactoryGirl.define do
   factory :sector do
     sequence(:public_id)
+    sequence(:correlation_id)
     name "Test Sector"
   end
 end

spec/lib/data_importer/activities_spec.rb

@@ -10,17 +10,17 @@
 "1000431","Nutritionist services","1002","Use CCTV systems"
       END
 
-      sector = FactoryGirl.create(:sector, public_id: 1000431)
+      sector = FactoryGirl.create(:sector, correlation_id: 1000431)
 
       importer = DataImporter::Activities.new(source)
       importer.run
 
-      imported_activity1 = Activity.find_by_public_id(362)
-      imported_activity1.public_id.should == 362
+      imported_activity1 = Activity.find_by_correlation_id(362)
+      imported_activity1.correlation_id.should == 362
       imported_activity1.name.should == "Practise as a dietitian "
 
-      imported_activity2 = Activity.find_by_public_id(1002)
-      imported_activity2.public_id.should == 1002
+      imported_activity2 = Activity.find_by_correlation_id(1002)
+      imported_activity2.correlation_id.should == 1002
       imported_activity2.name.should == "Use CCTV systems"
 
       sector.reload
@@ -34,12 +34,12 @@
 "1000431","Nutritionist services","362","Practise as a dietitian "
       END
 
-      sector = FactoryGirl.create(:sector, public_id: 1000431)
+      sector = FactoryGirl.create(:sector, correlation_id: 1000431)
 
       importer = DataImporter::Activities.new(source)
       importer.run
 
-      Activity.where(public_id: 362).length.should == 1
+      Activity.where(correlation_id: 362).length.should == 1
     end
   end
 

spec/lib/data_importer/licences_spec.rb

@@ -3,8 +3,8 @@
 
 describe DataImporter::Licences do
   before :each do
-    @sector = FactoryGirl.create(:sector, public_id: 1)
-    @activity = FactoryGirl.create(:activity, public_id: 1)
+    @sector = FactoryGirl.create(:sector, correlation_id: 1)
+    @activity = FactoryGirl.create(:activity, correlation_id: 1)
   end
 
   describe "clean import" do
@@ -17,8 +17,8 @@
       importer = DataImporter::Licences.new(source)
       importer.run
 
-      imported_licence = Licence.find_by_public_id(1)
-      imported_licence.public_id.should == 1
+      imported_licence = Licence.find_by_correlation_id(1)
+      imported_licence.correlation_id.should == 1
       imported_licence.name.should == "Licences to play music in a theatre (All UK)"
       imported_licence.regulation_area.should == "Copyright"
       imported_licence.da_england == true
@@ -32,8 +32,8 @@
       imported_licence_link.licence.should == imported_licence
       LicenceLink.all.length.should == 1
     end
-    it "should update the licence if one with the same public_id already exists" do
-      FactoryGirl.create(:licence, public_id: 1, name: "Test Name", da_england: false)
+    it "should update the licence if one with the same correlation_id already exists" do
+      FactoryGirl.create(:licence, correlation_id: 1, name: "Test Name", da_england: false)
 
       source = StringIO.new(<<-END)
 "SECTOR_OID","SECTOR","BUSINESSACT_ID","ACTIVITY_TITLE","LICENCE_OID","LICENCE","REGULATION_AREA","DA_ENGLAND","DA_SCOTLAND","DA_WALES","DA_NIRELAND","ALL_OF_UK"
@@ -43,8 +43,8 @@
       importer = DataImporter::Licences.new(source)
       importer.run
 
-      imported_licence = Licence.find_by_public_id(1)
-      imported_licence.public_id.should == 1
+      imported_licence = Licence.find_by_correlation_id(1)
+      imported_licence.correlation_id.should == 1
       imported_licence.name.should == "Licences to play music in a theatre (All UK)"
       imported_licence.da_england.should be_true
     end
@@ -58,7 +58,7 @@
       lambda do
         importer.run
       end.should raise_error
-      imported_licence = Licence.find_by_public_id(1)
+      imported_licence = Licence.find_by_correlation_id(1)
       imported_licence.should == nil
     end
     it "should fail early if the activity does not exist" do
@@ -71,13 +71,13 @@
       lambda do
         importer.run
       end.should raise_error
-      imported_licence = Licence.find_by_public_id(1)
+      imported_licence = Licence.find_by_correlation_id(1)
       imported_licence.should == nil
     end
     it "should add links for all layer3 sectors if a layer2 sector id is provided" do
-      sector1 = FactoryGirl.create(:sector, public_id: 2, layer2_id: 101)
-      sector2 = FactoryGirl.create(:sector, public_id: 3, layer2_id: 101)
-      sector3 = FactoryGirl.create(:sector, public_id: 4, layer2_id: 102)
+      sector1 = FactoryGirl.create(:sector, correlation_id: 2, layer2_id: 101)
+      sector2 = FactoryGirl.create(:sector, correlation_id: 3, layer2_id: 101)
+      sector3 = FactoryGirl.create(:sector, correlation_id: 4, layer2_id: 102)
 
       source = StringIO.new(<<-END)
 "SECTOR_OID","SECTOR","BUSINESSACT_ID","ACTIVITY_TITLE","LICENCE_OID","LICENCE","REGULATION_AREA","DA_ENGLAND","DA_SCOTLAND","DA_WALES","DA_NIRELAND","ALL_OF_UK"
@@ -95,9 +95,9 @@
       sectors.should_not include(sector3)
     end
     it "should add links for all layer3 sectors if a layer1 sector id is provided" do
-      sector1 = FactoryGirl.create(:sector, public_id: 2, layer1_id: 101)
-      sector2 = FactoryGirl.create(:sector, public_id: 3, layer1_id: 101)
-      sector3 = FactoryGirl.create(:sector, public_id: 4, layer1_id: 102)
+      sector1 = FactoryGirl.create(:sector, correlation_id: 2, layer1_id: 101)
+      sector2 = FactoryGirl.create(:sector, correlation_id: 3, layer1_id: 101)
+      sector3 = FactoryGirl.create(:sector, correlation_id: 4, layer1_id: 102)
 
       source = StringIO.new(<<-END)
 "SECTOR_OID","SECTOR","BUSINESSACT_ID","ACTIVITY_TITLE","LICENCE_OID","LICENCE","REGULATION_AREA","DA_ENGLAND","DA_SCOTLAND","DA_WALES","DA_NIRELAND","ALL_OF_UK"
@@ -116,7 +116,7 @@
     end
     it "should not create a new licence_link if it already exists" do
       require 'ruby-debug'
-      @licence = FactoryGirl.create(:licence, public_id: 1, name: "Test Name", da_england: false)
+      @licence = FactoryGirl.create(:licence, correlation_id: 1, name: "Test Name", da_england: false)
       licence_link = FactoryGirl.create(:licence_link, sector: @sector, activity: @activity, licence: @licence)
 
       source = StringIO.new(<<-END)
@@ -141,7 +141,7 @@
       importer = DataImporter::Licences.new(source)
       importer.run
 
-      imported_licence = Licence.find_by_public_id(1)
+      imported_licence = Licence.find_by_correlation_id(1)
       imported_licence.da_england.should == true
       imported_licence.da_scotland.should == true
       imported_licence.da_wales.should == true

spec/lib/data_importer/sectors_spec.rb

@@ -9,13 +9,13 @@
 "1000001","A0","Agriculture, forestry and fishing","1000002","A0.010","Agriculture","1000011","A0.010.090","Animal farming support services"
       END
 
-      Sector.find_by_public_id(1000011).should == nil
+      Sector.find_by_correlation_id(1000011).should == nil
 
       importer = DataImporter::Sectors.new(source)
       importer.run
 
-      imported_sector = Sector.find_by_public_id(1000011)
-      imported_sector.public_id.should == 1000011
+      imported_sector = Sector.find_by_correlation_id(1000011)
+      imported_sector.correlation_id.should == 1000011
       imported_sector.name.should == "Animal farming support services"
       imported_sector.layer1_id.should == 1000001
       imported_sector.layer2_id.should == 1000002
@@ -28,12 +28,12 @@
 "1000001","A0","Agriculture, forestry and fishing","1000002","A0.010","Agriculture","1000011","A0.010.090","Animal farming support services"
       END
 
-      Sector.find_by_public_id(1000011).should == nil
+      Sector.find_by_correlation_id(1000011).should == nil
 
       importer = DataImporter::Sectors.new(source)
       importer.run
 
-      Sector.where(public_id: 1000011).length.should == 1
+      Sector.where(correlation_id: 1000011).length.should == 1
     end
   end