Schedule a separate job for each group backup

app/jobs/export/backup_mitglieder_export_job.rb

@@ -3,28 +3,26 @@
 #  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
 #  hitobito_sac_cas and licensed under the Affero General Public License version 3
 #  or later. See the COPYING file at the top-level directory or at
-#  https://github.com/hitobito/hitobito_sac_cas
+#  https://github.com/hitobito/hitobito_sac_cas.
 
-class Export::BackupMitgliederExportJob < RecurringJob
-
-  run_every 1.day
+class Export::BackupMitgliederExportJob < BaseJob
+  self.parameters = [:group_id]
   self.use_background_job_logging = true
 
-  ROLE_TYPES_TO_BACKUP = [::Group::Sektion, ::Group::Ortsgruppe].freeze
-
-  def initialize
-    super
+  def initialize(group_id)
+    super()
+    @group = Group.find(group_id)
     @errors = []
   end
 
-  def perform_internal
-    relevant_groups.find_each do |group|
-      BackupMitgliederExport.new(group, sftp).call
-    rescue StandardError => e
-      error(self, e, group: group)
-      @errors << [group.id, e]
-      next
-    end
+  def perform
+    sftp.create_remote_dir(root_folder_path) unless sftp.directory?(root_folder_path)
+    sftp.create_remote_dir(folder_path) unless sftp.directory?(folder_path)
+
+    sftp.upload_file(csv, file_path)
+  rescue StandardError => e
+    error(self, e, group: @group)
+    @errors << [@group.id, e]
   end
 
   def log_results
@@ -35,8 +33,11 @@ def log_results
 
   private
 
-  def relevant_groups
-    Group.where(type: ROLE_TYPES_TO_BACKUP.map(&:sti_name))
+  def csv
+    @csv ||= begin
+               user_id = nil
+               SacCas::Export::MitgliederExportJob.new(user_id, @group.id).data
+             end
   end
 
   def sftp
@@ -47,7 +48,16 @@ def sftp_config
     Settings.sftp.config
   end
 
-  def next_run
-    interval.from_now.midnight + 5.minutes
+  def file_path
+    "#{folder_path}Adressen_#{@group.navision_id_padded}.csv"
+  end
+
+  def folder_path
+    "#{root_folder_path}#{@group.navision_id}/"
+  end
+
+  def root_folder_path
+    'sektionen/'
   end
+
 end

app/jobs/export/backup_mitglieder_schedule_job.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+class Export::BackupMitgliederScheduleJob < RecurringJob
+
+  run_every 1.day
+
+  ROLE_TYPES_TO_BACKUP = [Group::Sektion, Group::Ortsgruppe].freeze
+
+  def perform_internal
+    relevant_groups.find_each do |group|
+      Export::BackupMitgliederExportJob.new(group.id).enqueue!
+    end
+  end
+
+  private
+
+  def relevant_groups
+    Group.where(type: ROLE_TYPES_TO_BACKUP.map(&:sti_name))
+  end
+
+  def next_run
+    interval.from_now.midnight + 5.minutes
+  end
+end

lib/hitobito_sac_cas/wagon.rb

@@ -22,7 +22,7 @@ class Wagon < Rails::Engine
 
     config.to_prepare do
       JobManager.wagon_jobs += [
-        Export::BackupMitgliederExportJob,
+        Export::BackupMitgliederScheduleJob,
         PromoteNeuanmeldungenJob,
         Event::CloseApplicationsJob,
         Roles::TerminateTourenleiterJob

spec/domain/sftp_spec.rb

@@ -15,15 +15,14 @@
                         private_key: 'private key',
                         port: 22)
   end
+  let(:session) { instance_double('Net::SFTP::Session') }
 
   subject { Sftp.new(config) }
 
   context 'with password' do
     before { config.delete_field!(:private_key) }
 
     it 'creates connection with password credential' do
-      session = double
-
       expect(::Net::SFTP).to receive(:start)
                          .with('sftp.local', 'hitobito', { password: 'password',
                                                            non_interactive: true,
@@ -39,8 +38,6 @@
     before { config.delete_field!(:password) }
 
     it 'creates connection with private key' do
-      session = double
-
       expect(::Net::SFTP).to receive(:start)
                         .with('sftp.local', 'hitobito', { key_data: ['private key'],
                                                           non_interactive: true,

spec/jobs/export/backup_mitglieder_export_job_spec.rb

@@ -8,32 +8,9 @@
 require 'spec_helper'
 
 describe Export::BackupMitgliederExportJob do
-  subject(:job) { described_class.new }
-  let(:relevant_groups) { Group.where(type: [::Group::Sektion, ::Group::Ortsgruppe]) }
-
-  context 'rescheduling' do
-    it 'reschedules for tomorrow  at 5 minutes past midnight' do
-      job.perform
-      next_job = Delayed::Job.find_by("handler like '%BackupMitgliederExportJob%'")
-      expect(next_job.run_at).to eq Time.zone.tomorrow + 5.minutes
-    end
-  end
-
-  context 'perform' do
-    it 'only iterates over relevant groups' do
-      exporter = double
-      allow(exporter).to receive(:call)
-
-      relevant_groups.each do |group|
-        expect(BackupMitgliederExport).to receive(:new)
-          .with(group, an_instance_of(Sftp))
-          .and_return(exporter)
-
-      end
-
-      job.perform
-    end
-  end
+  subject(:job) { described_class.new(group.id).tap { _1.instance_variable_set(:@sftp, sftp) } }
+  let(:group) { groups(:bluemlisalp) }
+  let(:sftp) { double(:sftp) }
 
   context 'logging' do
     let(:notifications) { Hash.new {|h, k| h[k] = [] } }
@@ -58,20 +35,10 @@ def run_job(payload_object)
       exporter = double
       allow(exporter).to receive(:call)
 
-      error_group = relevant_groups.first
-
       error = Sftp::ConnectionError.new('permission denied')
 
-      expect(BackupMitgliederExport).to receive(:new)
-                                    .with(error_group, an_instance_of(Sftp))
-                                    .and_raise(error)
-      expect(job).to receive(:error).with(job, error, group: error_group)
-
-      (relevant_groups - [error_group]).each do |group|
-        expect(BackupMitgliederExport).to receive(:new)
-          .with(group, an_instance_of(Sftp))
-          .and_return(exporter)
-      end
+      expect(sftp).to receive(:directory?).and_raise(error)
+      expect(job).to receive(:error).with(job, error, group: group)
 
       job = subscribe { run_job(subject) }
 
@@ -99,9 +66,39 @@ def run_job(payload_object)
         group_id: nil,
         finished_at: an_instance_of(ActiveSupport::TimeWithZone),
         status: 'success',
-        payload: { errors: [[error_group.id, error]] },
+        payload: { errors: [[group.id, error]] },
         attempt: 0
       )
     end
   end
+
+  context 'perform' do
+    it 'tries to upload csv for group' do
+      csv_expectation = SacCas::Export::MitgliederExportJob.new(nil, group.id).data
+      root_folder_path_expectation = "sektionen/"
+      folder_path_expectation = "sektionen/1650/"
+      file_path_expectation = "sektionen/1650/Adressen_00001650.csv"
+
+      expect(sftp).to receive(:directory?).with(root_folder_path_expectation).and_return(true)
+      expect(sftp).to receive(:directory?).with(folder_path_expectation).and_return(true)
+      expect(sftp).to receive(:upload_file).with(csv_expectation, file_path_expectation)
+
+      job.perform
+    end
+
+    it 'tries to upload csv for group and create directories if not present' do
+      csv_expectation = SacCas::Export::MitgliederExportJob.new(nil, group.id).data
+      root_folder_path_expectation = "sektionen/"
+      folder_path_expectation = "sektionen/1650/"
+      file_path_expectation = "sektionen/1650/Adressen_00001650.csv"
+
+      expect(sftp).to receive(:directory?).with(root_folder_path_expectation).and_return(false)
+      expect(sftp).to receive(:create_remote_dir).with(root_folder_path_expectation)
+      expect(sftp).to receive(:directory?).with(folder_path_expectation).and_return(false)
+      expect(sftp).to receive(:create_remote_dir).with(folder_path_expectation)
+      expect(sftp).to receive(:upload_file).with(csv_expectation, file_path_expectation)
+
+      job.perform
+    end
+  end
 end

spec/jobs/export/backup_mitglieder_schedule_job_spec.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'spec_helper'
+
+describe Export::BackupMitgliederScheduleJob do
+  subject(:job) { described_class.new }
+  let(:relevant_groups) { Group.where(type: [Group::Sektion, Group::Ortsgruppe]) }
+
+  context 'rescheduling' do
+    it 'reschedules for tomorrow at 5 minutes past midnight' do
+      job.perform
+      next_job = Delayed::Job.find_by("handler like '%BackupMitgliederScheduleJob%'")
+      expect(next_job.run_at).to eq Time.zone.tomorrow + 5.minutes
+    end
+  end
+
+  context 'perform' do
+    it 'only iterates over relevant groups' do
+      relevant_groups.each do |group|
+        expect(Export::BackupMitgliederExportJob).to receive(:new)
+          .with(group.id)
+          .and_call_original
+      end
+
+      expect do
+        job.perform
+      end.to change { Delayed::Job.where("handler like '%BackupMitgliederExportJob%'").count }.by(relevant_groups.length)
+    end
+  end
+end