[modify] opendata harvest for ckan version up

.github/workflows/ruby-3.0.yml

@@ -63,6 +63,10 @@ jobs:
           fi
           echo $container_id
           echo "CONTAINER_ID=$container_id" >> $GITHUB_OUTPUT
+      - name: Prepare Ckan
+        run: |
+          docker pull ghcr.io/shirasagi/ckan:latest
+          docker tag ghcr.io/shirasagi/ckan:latest shirasagi/ckan:latest
       - name: Install pre requisites
         run: |
           sudo apt-get update

.github/workflows/ruby.yml

@@ -65,6 +65,10 @@ jobs:
           fi
           echo $container_id
           echo "CONTAINER_ID=$container_id" >> $GITHUB_OUTPUT
+      - name: Prepare Ckan
+        run: |
+          docker pull ghcr.io/shirasagi/ckan:latest
+          docker tag ghcr.io/shirasagi/ckan:latest shirasagi/ckan:latest
       - name: Install pre requisites
         run: |
           sudo apt-get update

app/controllers/opendata/dataset/harvest/exporter/group_settings_controller.rb → app/controllers/opendata/harvest/exporter/group_settings_controller.rb

@@ -1,4 +1,4 @@
-class Opendata::Dataset::Harvest::Exporter::GroupSettingsController < ApplicationController
+class Opendata::Harvest::Exporter::GroupSettingsController < ApplicationController
   include Cms::BaseFilter
   include Cms::CrudFilter
 

app/controllers/opendata/dataset/harvest/exporter/owner_org_settings_controller.rb → app/controllers/opendata/harvest/exporter/owner_org_settings_controller.rb

@@ -1,4 +1,4 @@
-class Opendata::Dataset::Harvest::Exporter::OwnerOrgSettingsController < ApplicationController
+class Opendata::Harvest::Exporter::OwnerOrgSettingsController < ApplicationController
   include Cms::BaseFilter
   include Cms::CrudFilter
 

app/controllers/opendata/dataset/harvest/exporters_controller.rb → app/controllers/opendata/harvest/exporters_controller.rb

@@ -1,4 +1,4 @@
-class Opendata::Dataset::Harvest::ExportersController < ApplicationController
+class Opendata::Harvest::ExportersController < ApplicationController
   include Cms::BaseFilter
   include Cms::CrudFilter
 
@@ -25,7 +25,7 @@ def export
     set_item
     return if request.get? || request.head?
 
-    Opendata::Harvest::HarvestDatasetsJob.bind(site_id: @cur_site, node_id: @cur_node).perform_later(exporter_id: @item.id)
-    flash.now[:notice] = "エクポート処理を開始しました。"
+    Opendata::Harvest::ExportJob.bind(site_id: @cur_site, node_id: @cur_node).perform_later(exporters: [@item.id])
+    flash.now[:notice] = I18n.t("ss.notice.started_export")
   end
 end

app/controllers/opendata/dataset/harvest/importer/category_settings_controller.rb → app/controllers/opendata/harvest/importer/category_settings_controller.rb

@@ -1,4 +1,4 @@
-class Opendata::Dataset::Harvest::Importer::CategorySettingsController < ApplicationController
+class Opendata::Harvest::Importer::CategorySettingsController < ApplicationController
   include Cms::BaseFilter
   include Cms::CrudFilter
 

app/controllers/opendata/dataset/harvest/importer/estat_category_settings_controller.rb → app/controllers/opendata/harvest/importer/estat_category_settings_controller.rb

@@ -1,4 +1,4 @@
-class Opendata::Dataset::Harvest::Importer::EstatCategorySettingsController < ApplicationController
+class Opendata::Harvest::Importer::EstatCategorySettingsController < ApplicationController
   include Cms::BaseFilter
   include Cms::CrudFilter
 

app/controllers/opendata/dataset/harvest/importer/reports_controller.rb → app/controllers/opendata/harvest/importer/reports_controller.rb

@@ -1,4 +1,4 @@
-class Opendata::Dataset::Harvest::Importer::ReportsController < ApplicationController
+class Opendata::Harvest::Importer::ReportsController < ApplicationController
   include Cms::BaseFilter
   include Cms::CrudFilter
 

app/controllers/opendata/dataset/harvest/importers_controller.rb → app/controllers/opendata/harvest/importers_controller.rb

@@ -1,12 +1,12 @@
-class Opendata::Dataset::Harvest::ImportersController < ApplicationController
+class Opendata::Harvest::ImportersController < ApplicationController
   include Cms::BaseFilter
   include Cms::CrudFilter
 
   model Opendata::Harvest::Importer
 
   navi_view "opendata/main/navi"
 
-  before_action :set_item, only: [:show, :edit, :update, :delete, :destroy, :import, :destroy_datasets]
+  before_action :set_item, only: [:show, :edit, :update, :delete, :destroy, :import, :purge]
   before_action :append_crumbs
 
   def append_crumbs
@@ -25,14 +25,14 @@ def index
   def import
     return if request.get? || request.head?
 
-    Opendata::Harvest::ImportDatasetsJob.bind(site_id: @cur_site, node_id: @cur_node).perform_later(@item.id)
-    flash.now[:notice] = I18n.t("opendata.errors.messages.import_started")
+    Opendata::Harvest::ImportJob.bind(site_id: @cur_site, node_id: @cur_node).perform_later(importers: [@item.id])
+    flash.now[:notice] = I18n.t("ss.notice.started_import")
   end
 
-  def destroy_datasets
+  def purge
     return if request.get? || request.head?
 
-    Opendata::Harvest::DestroyDatasetsJob.bind(site_id: @cur_site, node_id: @cur_node).perform_later(@item.id)
-    flash.now[:notice] = I18n.t("opendata.errors.messages.destroy_started")
+    Opendata::Harvest::PurgeJob.bind(site_id: @cur_site, node_id: @cur_node).perform_later(importers: [@item.id])
+    flash.now[:notice] = I18n.t("ss.notice.started_purge")
   end
 end

app/jobs/opendata/harvest/export_job.rb

@@ -0,0 +1,18 @@
+class Opendata::Harvest::ExportJob < Cms::ApplicationJob
+  def put_log(message)
+    Rails.logger.warn(message)
+    puts message
+  end
+
+  def perform(opts = {})
+    exporter_ids = opts[:exporters]
+
+    exporters = Opendata::Harvest::Exporter.site(site)
+    exporters = exporters.in(id: exporter_ids.map(&:to_i)) if exporter_ids
+
+    put_log("exporters: " + exporters.map { |item| "#{item.name}(#{item.id})" }.join(",") )
+    put_log("")
+
+    exporters.each(&:export)
+  end
+end

app/jobs/opendata/harvest/import_job.rb

@@ -0,0 +1,18 @@
+class Opendata::Harvest::ImportJob < Cms::ApplicationJob
+  def put_log(message)
+    Rails.logger.warn(message)
+    puts message
+  end
+
+  def perform(opts = {})
+    importer_ids = opts[:importers]
+
+    importers = Opendata::Harvest::Importer.site(site)
+    importers = importers.in(id: importer_ids.map(&:to_i)) if importer_ids
+
+    put_log("importers: " + importers.map { |item| "#{item.name}(#{item.id})" }.join(",") )
+    put_log("")
+
+    importers.each(&:import)
+  end
+end

app/jobs/opendata/harvest/purge_job.rb

@@ -0,0 +1,18 @@
+class Opendata::Harvest::PurgeJob < Cms::ApplicationJob
+  def put_log(message)
+    Rails.logger.warn(message)
+    puts message
+  end
+
+  def perform(opts = {})
+    importer_ids = opts[:importers]
+
+    importers = Opendata::Harvest::Importer.site(site)
+    importers = importers.in(id: importer_ids.map(&:to_i)) if importer_ids
+
+    put_log("importers: " + importers.map { |item| "#{item.name}(#{item.id})" }.join(",") )
+    put_log("")
+
+    importers.each(&:purge)
+  end
+end

app/jobs/opendata/harvest/run_job.rb

@@ -0,0 +1,24 @@
+class Opendata::Harvest::RunJob < Cms::ApplicationJob
+  def put_log(message)
+    Rails.logger.warn(message)
+    puts message
+  end
+
+  def perform(opts = {})
+    importer_ids = opts[:importers]
+    exporter_ids = opts[:exporters]
+
+    exporters = Opendata::Harvest::Exporter.site(site)
+    exporters = exporters.in(id: exporter_ids.map(&:to_i)) if exporter_ids
+
+    importers = Opendata::Harvest::Importer.site(site)
+    importers = importers.in(id: importer_ids.map(&:to_i)) if importer_ids
+
+    put_log("importers: " + importers.map { |item| "#{item.name}(#{item.id})" }.join(",") )
+    put_log("exporters: " + exporters.map { |item| "#{item.name}(#{item.id})" }.join(",") )
+    put_log("")
+
+    importers.each(&:import)
+    exporters.each(&:export)
+  end
+end

app/models/concerns/opendata/addon/harvest/importer.rb

@@ -9,6 +9,11 @@ module Importer
     EXTERNAL_RESOUCE_FORMAT = %w(html htm).freeze
 
     def import
+      if !enabled?
+        put_log("not enabled #{source_url}")
+        return
+      end
+
       if api_type == "ckan"
         import_from_ckan_api
       elsif api_type == "shirasagi_api"
@@ -18,7 +23,7 @@ def import
       end
     end
 
-    def destroy_imported_datasets
+    def purge
       dataset_ids = ::Opendata::Dataset.site(site).node(node).where("$or" => [
           { harvest_api_type: api_type, harvest_host: source_host },
           { harvest_importer_id: id }

app/models/concerns/opendata/addon/harvest/resource.rb

@@ -12,6 +12,7 @@ module Opendata::Addon::Harvest::Resource
     field :harvest_imported, type: DateTime, default: nil
     field :harvest_imported_url, type: String, default: nil
     field :harvest_imported_attributes, type: Hash, default: {}
+    field :harvest_last_modified, type: DateTime, default: nil
 
     field :harvest_host, type: String, default: nil
     field :harvest_api_type, type: String, default: nil

app/models/concerns/opendata/harvest/ckan_api_export_initializer.rb

@@ -36,12 +36,10 @@ def dataset_purge
 
         put_log "#{idx}-#{r_idx} : resource_delete #{resource_id}"
         package.resource_delete(resource_id, api_key)
-        sleep 1
       end
 
       put_log "#{idx} : dataset_purge #{name} #{id}"
       package.dataset_purge(id, api_key)
-      sleep 1
     end
 
     self.dataset_relations.destroy_all

app/models/concerns/opendata/harvest/ckan_api_exporter.rb

@@ -8,6 +8,11 @@ def put_log(message)
   end
 
   def export
+    if !enabled?
+      put_log "not enabled #{url}"
+      return
+    end
+
     put_log "export to #{url} (Ckan API)"
 
     @package = ::Opendata::Harvest::CkanPackage.new(url)
@@ -23,7 +28,6 @@ def export
       next unless dataset
 
       dataset_relation = export_dataset(dataset, d_idx)
-      sleep 1
       next unless dataset_relation
 
       @dataset_relations[dataset_relation.rel_id] = dataset_relation
@@ -34,7 +38,6 @@ def export
       resources_relations = []
       dataset.resources.each_with_index do |resource, r_idx|
         resources_relation = export_resource(resource, dataset_relation, exported_resources, d_idx, r_idx)
-        sleep 1
         next unless resources_relation
 
         @resource_relations[resources_relation.rel_id] = resources_relation
@@ -104,8 +107,7 @@ def export_resource(resource, dataset_relation, exported_resources, d_idx, r_idx
         attributes = {
           uuid: resource.uuid,
           revision_id: resource.revision_id,
-          rel_id: resource_relation.rel_id,
-          rel_revision_id: resource.revision_id
+          rel_id: resource_relation.rel_id
         }
       else
         put_log "#{d_idx}-#{r_idx} : update resource #{resource.name} #{resource.uuid}"
@@ -121,8 +123,7 @@ def export_resource(resource, dataset_relation, exported_resources, d_idx, r_idx
         attributes = {
           uuid: resource.uuid,
           revision_id: resource.revision_id,
-          rel_id: result["id"],
-          rel_revision_id: result["revision_id"]
+          rel_id: result["id"]
         }
       end
       resource_relation.attributes = attributes
@@ -144,8 +145,7 @@ def export_resource(resource, dataset_relation, exported_resources, d_idx, r_idx
         rel_dataset: dataset_relation,
         uuid: resource.uuid,
         revision_id: resource.revision_id,
-        rel_id: result["id"],
-        rel_revision_id: result["revision_id"]
+        rel_id: result["id"]
       }
       resource_relation.attributes = attributes
       resource_relation.save!
@@ -275,10 +275,15 @@ def dataset_update_params(dataset)
     params
   end
 
+  def resource_url_param(resource)
+    return resource.file.filename if resource.source_url.blank?
+    resource.source_url.sub(/\?.*$/, "")
+  end
+
   def resource_create_params(resource)
     params = {
       name: resource.name,
-      url: (resource.source_url.presence || resource.file.filename),
+      url: resource_url_param(resource),
       description: resource.text,
       format: resource.format
     }
@@ -288,7 +293,7 @@ def resource_create_params(resource)
   def resource_update_params(resource)
     params = {
       name: resource.name,
-      url: (resource.source_url.presence || resource.file.filename),
+      url: resource_url_param(resource),
       description: resource.text,
       format: resource.format
     }