Update NL municipalities to 2019

db/migrate/20240306142351_netherlands_municipalities_2019_update_step_1.rb

@@ -0,0 +1,65 @@
+class NetherlandsMunicipalities2019UpdateStep1 < ActiveRecord::Migration[5.0]
+  def self.up
+    directory    = Rails.root.join('db/migrate/20240306142351_netherlands_municipalities_2019_update_step_1')
+    data_path    = directory.join('data.csv')
+    commits_path = directory.join('commits.yml')
+    datasets     = []
+
+    # By default, CSVImporter only allows updating existing datasets. If the
+    # migration is adding a new dataset, add the `create_missing_datasets`
+    # keyword argument. For example:
+    #
+    #   CSVImporter.run(data_path, commits_path, create_missing_datasets: true) do |row, runner|
+    #     # ...
+    #   end
+    #
+    CSVImporter.run(data_path, commits_path) do |row, runner|
+      print "Updating #{row['geo_id']}... "
+      commits = runner.call
+
+      if commits.any?
+        datasets.push(find_dataset(commits))
+        puts 'done!'
+      else
+        puts 'nothing to change!'
+      end
+    end
+
+    sleep(1)
+    puts
+    puts "Updated #{datasets.length} datasets with the following IDs:"
+    puts "  #{datasets.map(&:id).join(',')}"
+  rescue ActiveRecord::RecordInvalid => e
+    if e.record.is_a?(Commit) && e.record.errors['dataset_edits.value'].any?
+      warn('')
+      warn('-' * 80)
+      warn('The following errors occurred while processing CSV rows:')
+      warn('')
+
+      # Grab all the errors from individual datasets to show those instead. This is typically
+      # the case when a CSV specifies a value that is not allowed to be edited.
+      e.record
+        .dataset_edits
+        .reject(&:valid?)
+        .each do |edit|
+          edit.errors.each do |field, msg|
+            warn("* #{edit.commit.dataset.geo_id}: #{edit.key}: #{field} #{msg}")
+          end
+        end
+
+      warn('-' * 80)
+    end
+
+    raise e
+  end
+
+  def self.down
+    raise ActiveRecord::IrreversibleMigration
+  end
+
+  def find_dataset(commits)
+    commits.each do |commit|
+      return commit.dataset if commit&.dataset
+    end
+  end
+end

db/migrate/20240306142351_netherlands_municipalities_2019_update_step_1/new_municipalities.csv

@@ -0,0 +1,2 @@
+geo_id,lv_net_total_costs_present,mv_net_total_costs_present,hv_net_total_costs_present,lv_mv_trafo_total_costs_present,mv_hv_trafo_total_costs_present,households_final_demand_electricity_demand,input_buildings_electricity_demand,buildings_final_demand_network_gas_demand,agriculture_final_demand_electricity_demand,agriculture_final_demand_network_gas_demand,input_energy_chp_local_engine_ht_network_gas_production,input_agriculture_final_demand_steam_hot_water_demand,input_industry_ict_electricity_demand,input_industry_chemical_other_electricity_demand,input_industry_food_electricity_demand,input_industry_metal_other_electricity_demand,input_industry_paper_electricity_demand,input_industry_other_electricity_demand,input_industry_chemical_other_network_gas_demand,input_industry_food_network_gas_demand,input_industry_metal_other_network_gas_demand,input_industry_paper_network_gas_demand,input_industry_other_network_gas_demand,energy_power_sector_own_use_electricity_demand,energy_power_hv_network_loss_demand,energy_distribution_greengas_demand,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+GM1979,2019,46051,23185,2318,71,2,0.001361637,0.000115827,0.016169861,0.00144568,0.000114515,0.000887955,0.001213229,0.000671219,0.000377142,0.003312733,0.005297946,0,0.07560158,0.027649089,0,0.045037639,0.003009943,0.026561851,0.008365145,30060,17727389.1,15234475,8952331.493,6315382.365,5318216.728,0,1.656970818,0,0.0945,0.0733,0.0743,0.072,0.0774,0.102777778,0.03597,0.0715,0.1012,0.0565,0.0667,0.0565,0.1052,32455496.46,413.4278912,500.1613214,9.205010817,1.657405447,26.96853948,31.03146052,0.112092817,0.058695652,0,0,302,0,0,0,0,0,8.214273736,11,0,0,0,0,0,0,0.956217869,0,0,0.035844095,0.007938037,0.160225149,0.053174884,0.709685802,0.076914165,0.425508436,0,0.096439891,0.130547403,0,0.347504271,0,1,0,0,1,0,0,0,0.767969802,0.232030198,0.862896085,0,0.137103915,0.106572736,0.893427264,1,491955.99,166290.19,0,203.0762325,990.645,48.01,0,0,99,0,0.98989899,0.01010101,0,0,1,0,1,0.743769786,0.034247382,0.011492672,0.032963105,0.104644779,0.072882275,0.02049842,0,0.151879838,0.827621742,0.844941736,0.098360656,0.131147541,0.081967213,0.295081967,0.114754098,0.131147541,0.06557377,0.081967213,0.713740458,0.188931298,0.097328244,0.722722307,0.257053425,0.020224268,0.187672747,0.791873964,0.020453289,0.675787227,0.238595983,0.074306809,0.01130998,0.963885599,0.000321773,0.035792628,177.7960883,337.8321,4288,12,0,0,0,1,0,1,0,0.486211623,0.125054127,0.356446796,0.032287454,0.001659312,0,0.998340688,0,1,0.978823114,0,0.021176886,0.012888523,0.031905638,0.955205839,1,0,0,0,1,29.33,49.48144385,59.57590275,0.485377279,0,0,0,0.90724725,0,0,42,53.5,42,39,65,44,3942,3650,8500,0,0,0,0,0,0,1,8.2116,3124.754725,6.2935644,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,43.3,30.5,43,47,24.75,35,40,15,18.3,47,72,92.4,72,0,0,33.445,41.472,100,8300,7900,0,0,0,0,0,0,0,479.6150646,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1619,0,2666,0,0,0,0,1119,0,0,95.66,828,0,0,0,0.58,0.42,0,3,0,0,4530,3925,4000,2005,5000,4100,750,1590,2644,2069.333333,3733.54232,867,43.3,30.5,41,40,40,50,18.3,47,20.45,19.95,43,14,40,15,35,15,72,92.4,109.593,91.5,72,80,37.3,49,45.3,60,38,48,0,36.93,36.93,36,25,34,36.1,46,42,45,40,40,30.5

db/migrate/20240306142353_netherlands_municipalities_2019_update_step_2.rb

@@ -0,0 +1,65 @@
+class NetherlandsMunicipalities2019UpdateStep2 < ActiveRecord::Migration[5.0]
+  def self.up
+    directory    = Rails.root.join('db/migrate/20240306142353_netherlands_municipalities_2019_update_step_2')
+    data_path    = directory.join('data.csv')
+    commits_path = directory.join('commits.yml')
+    datasets     = []
+
+    # By default, CSVImporter only allows updating existing datasets. If the
+    # migration is adding a new dataset, add the `create_missing_datasets`
+    # keyword argument. For example:
+    #
+    #   CSVImporter.run(data_path, commits_path, create_missing_datasets: true) do |row, runner|
+    #     # ...
+    #   end
+    #
+    CSVImporter.run(data_path, commits_path) do |row, runner|
+      print "Updating #{row['geo_id']}... "
+      commits = runner.call
+
+      if commits.any?
+        datasets.push(find_dataset(commits))
+        puts 'done!'
+      else
+        puts 'nothing to change!'
+      end
+    end
+
+    sleep(1)
+    puts
+    puts "Updated #{datasets.length} datasets with the following IDs:"
+    puts "  #{datasets.map(&:id).join(',')}"
+  rescue ActiveRecord::RecordInvalid => e
+    if e.record.is_a?(Commit) && e.record.errors['dataset_edits.value'].any?
+      warn('')
+      warn('-' * 80)
+      warn('The following errors occurred while processing CSV rows:')
+      warn('')
+
+      # Grab all the errors from individual datasets to show those instead. This is typically
+      # the case when a CSV specifies a value that is not allowed to be edited.
+      e.record
+        .dataset_edits
+        .reject(&:valid?)
+        .each do |edit|
+          edit.errors.each do |field, msg|
+            warn("* #{edit.commit.dataset.geo_id}: #{edit.key}: #{field} #{msg}")
+          end
+        end
+
+      warn('-' * 80)
+    end
+
+    raise e
+  end
+
+  def self.down
+    raise ActiveRecord::IrreversibleMigration
+  end
+
+  def find_dataset(commits)
+    commits.each do |commit|
+      return commit.dataset if commit&.dataset
+    end
+  end
+end

db/migrate/20240306142353_netherlands_municipalities_2019_update_step_2/commits.yml

@@ -0,0 +1,32 @@
+---
+- fields: 
+  - lv_net_total_costs_present
+  - mv_net_total_costs_present
+  - hv_net_total_costs_present
+  - lv_mv_trafo_total_costs_present
+  - mv_hv_trafo_total_costs_present
+  - households_final_demand_electricity_demand
+  - input_buildings_electricity_demand
+  - buildings_final_demand_network_gas_demand
+  - agriculture_final_demand_electricity_demand
+  - agriculture_final_demand_network_gas_demand
+  - input_energy_chp_local_engine_ht_network_gas_production
+  - input_agriculture_final_demand_steam_hot_water_demand
+  - input_industry_ict_electricity_demand
+  - input_industry_chemical_other_electricity_demand
+  - input_industry_food_electricity_demand
+  - input_industry_metal_other_electricity_demand
+  - input_industry_paper_electricity_demand
+  - input_industry_other_electricity_demand
+  - input_industry_chemical_other_network_gas_demand
+  - input_industry_food_network_gas_demand
+  - input_industry_metal_other_network_gas_demand
+  - input_industry_paper_network_gas_demand
+  - input_industry_other_network_gas_demand
+  - energy_power_sector_own_use_electricity_demand
+  - energy_power_hv_network_loss_demand
+  - energy_distribution_greengas_demand
+  message:
+   "Schatting Quintel. Geen data beschikbaar voor 2019. De schatting is gebaseerd op het totaal energie/elektriciteits/gasgebruik in de gemeente (zoals gerapporteerd door Klimaatmonitor) en/of op data uit voorgaande jaren. Zie: https://klimaatmonitor.databank.nl/Jive?workspace_guid=59c3f31c-ab95-420f-b592-5c90d934d2ef 
+En: https://klimaatmonitor.databank.nl/Jive?workspace_guid=04e40306-c921-4a5f-8b2f-bdc258886267"
+