Merge pull request #554 from hitobito/feature/sac_272_export_sftp_upload

Backup mitglieder export via sftp

app/domain/sftp.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'net/sftp'
+
+class Sftp
+  ConnectionError = Class.new(StandardError)
+
+  def initialize(config)
+    @config = config
+  end
+
+  def upload_file(data, file_path)
+    create_missing_directories(file_path)
+
+    handle = connection.open!(file_path, 'w')
+    connection.write!(handle, 0, data)
+    connection.close(handle)
+  end
+
+  def create_remote_dir(name)
+    connection.mkdir!(name)
+  end
+
+  def directory?(name)
+    connection.file.directory?(name)
+  rescue
+    false
+  end
+
+  private
+
+  def create_missing_directories(file_path)
+    Pathname.new(file_path).dirname.descend do |directory_path|
+      create_remote_dir(directory_path.to_s) unless directory?(directory_path.to_s)
+    end
+  end
+
+  def connection
+    @connection ||= Net::SFTP.start(@config.host, @config.user, options).tap(&:connect!)
+  rescue => e
+    raise ConnectionError.new(e)
+  end
+
+  def options
+    credentials = if @config.private_key.present?
+                    { key_data: [@config.private_key] }
+                  else
+                    { password: @config.password }
+                  end
+    credentials.merge(non_interactive: true, port: @config.port).compact
+  end
+end

app/jobs/export/backup_mitglieder_export_job.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+class Export::BackupMitgliederExportJob < BaseJob
+  self.parameters = [:group_id]
+  self.use_background_job_logging = true
+
+  def initialize(group_id)
+    super()
+    @group = Group.find(group_id)
+    @errors = []
+  end
+
+  def perform
+    sftp.upload_file(csv, file_path)
+  rescue => e
+    error(self, e, group: @group)
+    @errors << [@group.id, e]
+  end
+
+  def log_results
+    {
+      errors: @errors
+    }
+  end
+
+  private
+
+  def csv
+    @csv ||= begin
+               user_id = nil
+               SacCas::Export::MitgliederExportJob.new(user_id, @group.id).data
+             end
+  end
+
+  def sftp
+    @sftp ||= Sftp.new(sftp_config)
+  end
+
+  def sftp_config
+    Settings.sftp.config
+  end
+
+  def file_path
+    "sektionen/#{@group.navision_id}/Adressen_#{@group.navision_id_padded}.csv"
+  end
+end

app/jobs/export/backup_mitglieder_schedule_job.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+class Export::BackupMitgliederScheduleJob < RecurringJob
+
+  run_every 1.day
+
+  ROLE_TYPES_TO_BACKUP = [Group::Sektion, Group::Ortsgruppe].freeze
+
+  def perform_internal
+    relevant_groups.find_each do |group|
+      Export::BackupMitgliederExportJob.new(group.id).enqueue!
+    end
+  end
+
+  private
+
+  def relevant_groups
+    Group.where(type: ROLE_TYPES_TO_BACKUP.map(&:sti_name))
+  end
+
+  def next_run
+    interval.from_now.midnight + 5.minutes
+  end
+end

config/settings.yml

@@ -33,6 +33,12 @@ event:
 # to this address to set a password.
 root_email: hitobito-sac-cas@puzzle.ch
 
+sftp:
+  config:
+    <% if ENV['RAILS_SFTP_CONFIG'].present? %>
+      <%= "{ #{ENV['RAILS_SFTP_CONFIG']} }" %>
+    <% end %>
+
 people:
   people_managers:
     enabled: true

hitobito_sac_cas.gemspec

@@ -17,5 +17,7 @@ Gem::Specification.new do |s|
   s.files = Dir['{app,config,db,lib}/**/*'] + ['Rakefile']
   s.test_files = Dir['test/**/*']
   s.add_dependency 'hitobito_youth'
+  s.add_dependency 'net-ssh', '~> 7.0.0.beta1' # TODO: remove once net-sftp 3 updates ssh 7
+  s.add_dependency 'net-sftp'
   # rubocop:enable SingleSpaceBeforeFirstArg
 end

lib/hitobito_sac_cas/wagon.rb

@@ -22,6 +22,7 @@ class Wagon < Rails::Engine
 
     config.to_prepare do # rubocop:disable Metrics/BlockLength
       JobManager.wagon_jobs += [
+        Export::BackupMitgliederScheduleJob,
         PromoteNeuanmeldungenJob,
         Event::CloseApplicationsJob,
         Roles::TerminateTourenleiterJob

spec/domain/sftp_spec.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'spec_helper'
+
+describe Sftp do
+  let(:config) do
+    Config::Options.new(host: 'sftp.local',
+                        user: 'hitobito',
+                        password: 'password',
+                        private_key: 'private key',
+                        port: 22)
+  end
+  let(:session) { instance_double('Net::SFTP::Session') }
+
+  subject(:sftp) { Sftp.new(config) }
+
+  context 'with password' do
+    before { config.delete_field!(:private_key) }
+
+    it 'creates connection with password credential' do
+      expect(::Net::SFTP).to receive(:start)
+                         .with('sftp.local', 'hitobito', { password: 'password',
+                                                           non_interactive: true,
+                                                           port: 22 })
+                       .and_return(session)
+      expect(session).to receive(:connect!)
+
+      subject.send(:connection)
+    end
+  end
+
+  context 'with private key' do
+    before { config.delete_field!(:password) }
+
+    it 'creates connection with private key' do
+      expect(::Net::SFTP).to receive(:start)
+                        .with('sftp.local', 'hitobito', { key_data: ['private key'],
+                                                          non_interactive: true,
+                                                          port: 22 })
+                       .and_return(session)
+      expect(session).to receive(:connect!)
+
+      subject.send(:connection)
+    end
+  end
+
+  context 'with private key and password' do
+    it 'creates connection with private key' do
+      expect(::Net::SFTP).to receive(:start)
+                         .with('sftp.local', 'hitobito', { key_data: ['private key'],
+                                                           non_interactive: true,
+                                                           port: 22 })
+                       .and_return(session)
+      expect(session).to receive(:connect!)
+
+      subject.send(:connection)
+    end
+  end
+
+  context '#upload_file' do
+    it 'tries to upload csv for group and does not create directories if present' do
+      group = groups(:bluemlisalp)
+
+      root_folder_path = "sektionen"
+      folder_path = "sektionen/1650"
+      file_path = "sektionen/1650/Adressen_00001650.csv"
+
+      expect(sftp).to receive(:directory?).with(root_folder_path).and_return(true)
+      expect(sftp).to_not receive(:create_remote_dir).with(root_folder_path)
+      expect(sftp).to receive(:directory?).with(folder_path).and_return(true)
+      expect(sftp).to_not receive(:create_remote_dir).with(folder_path)
+
+      expect(::Net::SFTP).to receive(:start).and_return(session)
+      expect(session).to receive(:connect!)
+      expect(session).to receive(:open!).with(file_path, 'w').and_return('handler')
+      expect(session).to receive(:write!).with('handler', 0, 'data')
+      expect(session).to receive(:close)
+
+      sftp.upload_file('data', file_path)
+    end
+
+    it 'tries to upload csv for group and create directories if not present' do
+      group = groups(:bluemlisalp)
+
+      root_folder_path = "sektionen"
+      folder_path = "sektionen/1650"
+      file_path = "sektionen/1650/Adressen_00001650.csv"
+
+      expect(sftp).to receive(:directory?).with(root_folder_path).and_return(false)
+      expect(sftp).to receive(:create_remote_dir).with(root_folder_path)
+      expect(sftp).to receive(:directory?).with(folder_path).and_return(false)
+      expect(sftp).to receive(:create_remote_dir).with(folder_path)
+
+      expect(::Net::SFTP).to receive(:start).and_return(session)
+      expect(session).to receive(:connect!)
+      expect(session).to receive(:open!).with(file_path, 'w').and_return('handler')
+      expect(session).to receive(:write!).with('handler', 0, 'data')
+      expect(session).to receive(:close)
+
+      sftp.upload_file('data', file_path)
+    end
+  end
+end

spec/jobs/export/backup_mitglieder_export_job_spec.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'spec_helper'
+
+describe Export::BackupMitgliederExportJob do
+  subject(:job) { described_class.new(group.id).tap { _1.instance_variable_set(:@sftp, sftp) } }
+  let(:group) { groups(:bluemlisalp) }
+  let(:sftp) { double(:sftp) }
+
+  context 'logging' do
+    let(:notifications) { Hash.new {|h, k| h[k] = [] } }
+
+    def subscribe
+      callback = lambda do |name, started, finished, unique_id, payload|
+        notifications[name] <<
+          OpenStruct.new(name: name, started: started, finished: finished, unique_id: unique_id, payload: payload)
+      end
+      ActiveSupport::Notifications.subscribed(callback, /\w+\.background_job/) do
+        yield
+      end
+    end
+
+    def run_job(payload_object)
+      payload_object.enqueue!.tap do |job_instance|
+        Delayed::Worker.new.run(job_instance)
+      end
+    end
+
+    it 'logs any type of error raised and continues' do
+      exporter = double
+      allow(exporter).to receive(:call)
+
+      error = Sftp::ConnectionError.new('permission denied')
+
+      expect(sftp).to receive(:upload_file).and_raise(error)
+      expect(job).to receive(:error).with(job, error, group: group)
+
+      job = subscribe { run_job(subject) }
+
+      expect(notifications.keys).to match_array [
+        "job_started.background_job",
+        "job_finished.background_job"
+      ]
+
+      expect(notifications["job_started.background_job"]).to have(1).item
+      expect(notifications["job_finished.background_job"]).to have(1).item
+
+      started_attrs = notifications["job_started.background_job"].first[:payload]
+      expect(started_attrs).to match(
+        job_id: job.id,
+        job_name: described_class.name,
+        group_id: nil,
+        started_at: an_instance_of(ActiveSupport::TimeWithZone),
+        attempt: 0
+      )
+
+      finished_attrs = notifications["job_finished.background_job"].first[:payload]
+      expect(finished_attrs).to match(
+        job_id: job.id,
+        job_name: described_class.name,
+        group_id: nil,
+        finished_at: an_instance_of(ActiveSupport::TimeWithZone),
+        status: 'success',
+        payload: { errors: [[group.id, error]] },
+        attempt: 0
+      )
+    end
+  end
+
+  context 'perform' do
+    it 'tries to upload csv for group' do
+      csv_expectation = SacCas::Export::MitgliederExportJob.new(nil, group.id).data
+      file_path_expectation = "sektionen/1650/Adressen_00001650.csv"
+
+      expect(sftp).to receive(:upload_file).with(csv_expectation, file_path_expectation)
+
+      job.perform
+    end
+  end
+end

spec/jobs/export/backup_mitglieder_schedule_job_spec.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'spec_helper'
+
+describe Export::BackupMitgliederScheduleJob do
+  subject(:job) { described_class.new }
+  let(:relevant_groups) { Group.where(type: [Group::Sektion, Group::Ortsgruppe]) }
+
+  context 'rescheduling' do
+    it 'reschedules for tomorrow at 5 minutes past midnight' do
+      job.perform
+      next_job = Delayed::Job.find_by("handler like '%BackupMitgliederScheduleJob%'")
+      expect(next_job.run_at).to eq Time.zone.tomorrow + 5.minutes
+    end
+  end
+
+  context 'perform' do
+    it 'only iterates over relevant groups' do
+      relevant_groups.each do |group|
+        expect(Export::BackupMitgliederExportJob).to receive(:new)
+          .with(group.id)
+          .and_call_original
+      end
+
+      expect do
+        job.perform
+      end.to change { Delayed::Job.where("handler like '%BackupMitgliederExportJob%'").count }.by(relevant_groups.length)
+    end
+  end
+end