Backup mitglieder export via sftp

Refs: #272

app/domain/backup_mitglieder_export.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+class BackupMitgliederExport
+
+  def initialize(group, sftp)
+    @group = group
+    @sftp = sftp
+  end
+
+  def call
+    @sftp.create_remote_dir(root_folder_path) unless @sftp.directory?(root_folder_path)
+    @sftp.create_remote_dir(folder_path) unless @sftp.directory?(folder_path)
+
+    @sftp.upload_file(csv, file_path)
+  end
+
+  private
+
+  def csv
+    @csv ||= begin
+               user_id = nil
+               SacCas::Export::MitgliederExportJob.new(user_id, @group.id).data
+             end
+  end
+
+  def file_path
+    "#{folder_path}Adressen_#{@group.navision_id_padded}.csv"
+  end
+
+  def folder_path
+    "#{root_folder_path}#{@group.navision_id}/"
+  end
+
+  def root_folder_path
+    'sektionen/'
+  end
+
+end

app/domain/sftp.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'net/sftp'
+
+class Sftp
+  ConnectionError = Class.new(StandardError)
+
+  def initialize(config)
+    @config = config
+  end
+
+  def upload_file(data, file_path)
+    handle = @connection.open!(file_path, 'w')
+    @connection.write!(handle, 0, data)
+    @connection.close(handle)
+  end
+
+  def create_remote_dir(name)
+    connection.mkdir!(name)
+  end
+
+  def directory?(name)
+    connection.file.directory?(name)
+  rescue
+    false
+  end
+
+  private
+
+  def connection
+    @connection ||= Net::SFTP.start(@config.host, @config.user, options).tap(&:connect!)
+  rescue => e
+    raise ConnectionError.new(e)
+  end
+
+  def options
+    credentials = if @config.private_key.present?
+                    { key_data: [@config.private_key] }
+                  else
+                    { password: @config.password }
+                  end
+    credentials.merge(non_interactive: true, port: @config.port).compact
+  end
+end

app/jobs/export/backup_mitglieder_export_job.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+class Export::BackupMitgliederExportJob < RecurringJob
+
+  run_every 1.day
+  self.use_background_job_logging = true
+
+  ROLE_TYPES_TO_BACKUP = [::Group::Sektion, ::Group::Ortsgruppe].freeze
+
+  def initialize
+    super
+    @errors = []
+  end
+
+  def perform_internal
+    relevant_groups.find_each do |group|
+      BackupMitgliederExport.new(group, sftp).call
+    rescue StandardError => e
+      error(self, e, group: group)
+      @errors << [group.id, e]
+      next
+    end
+  end
+
+  def log_results
+    {
+      errors: @errors
+    }
+  end
+
+  private
+
+  def relevant_groups
+    Group.where(type: ROLE_TYPES_TO_BACKUP.map(&:sti_name))
+  end
+
+  def sftp
+    @sftp ||= Sftp.new(sftp_config)
+  end
+
+  def sftp_config
+    Settings.sftp.config
+  end
+
+  def next_run
+    interval.from_now.midnight + 5.minutes
+  end
+end

config/settings.yml

@@ -34,6 +34,12 @@ event:
 # to this address to set a password.
 root_email: hitobito-sac-cas@puzzle.ch
 
+sftp:
+  config:
+    <% if ENV['RAILS_SFTP_CONFIG'].present? %>
+      <%= "{ #{ENV['RAILS_SFTP_CONFIG']} }" %>
+    <% end %>
+
 people:
   people_managers:
     enabled: true

hitobito_sac_cas.gemspec

@@ -17,5 +17,7 @@ Gem::Specification.new do |s|
   s.files = Dir['{app,config,db,lib}/**/*'] + ['Rakefile']
   s.test_files = Dir['test/**/*']
   s.add_dependency 'hitobito_youth'
+  s.add_dependency 'net-ssh', '~> 7.0.0.beta1' # TODO: remove once net-sftp 3 updates ssh 7
+  s.add_dependency 'net-sftp'
   # rubocop:enable SingleSpaceBeforeFirstArg
 end

spec/domain/backup_mitglieder_export_spec.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'spec_helper'
+
+describe BackupMitgliederExport do
+  let(:group) { groups(:bluemlisalp) }
+  let(:sftp) { double(:sftp) }
+
+  let(:export) { BackupMitgliederExport.new(group, sftp) }
+
+  it 'tries to upload csv for group' do
+    csv_expectation = SacCas::Export::MitgliederExportJob.new(nil, group.id).data
+    root_folder_path_expectation = "sektionen/"
+    folder_path_expectation = "sektionen/1650/"
+    file_path_expectation = "sektionen/1650/Adressen_00001650.csv"
+
+    expect(sftp).to receive(:directory?).with(root_folder_path_expectation).and_return(true)
+    expect(sftp).to receive(:directory?).with(folder_path_expectation).and_return(true)
+    expect(sftp).to receive(:upload_file).with(csv_expectation, file_path_expectation)
+
+    export.call
+  end
+
+  it 'tries to upload csv for group and create directories if not present' do
+    csv_expectation = SacCas::Export::MitgliederExportJob.new(nil, group.id).data
+    root_folder_path_expectation = "sektionen/"
+    folder_path_expectation = "sektionen/1650/"
+    file_path_expectation = "sektionen/1650/Adressen_00001650.csv"
+
+    expect(sftp).to receive(:directory?).with(root_folder_path_expectation).and_return(false)
+    expect(sftp).to receive(:create_remote_dir).with(root_folder_path_expectation)
+    expect(sftp).to receive(:directory?).with(folder_path_expectation).and_return(false)
+    expect(sftp).to receive(:create_remote_dir).with(folder_path_expectation)
+    expect(sftp).to receive(:upload_file).with(csv_expectation, file_path_expectation)
+
+    export.call
+  end
+end

spec/domain/sftp_spec.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'spec_helper'
+
+describe Sftp do
+  let(:config) do
+    Config::Options.new(host: 'sftp.local',
+                        user: 'hitobito',
+                        password: 'password',
+                        private_key: 'private key',
+                        port: 22)
+  end
+
+  subject { Sftp.new(config) }
+
+  context 'with password' do
+    before { config.delete_field!(:private_key) }
+
+    it 'creates connection with password credential' do
+      session = double
+
+      expect(::Net::SFTP).to receive(:start)
+                         .with('sftp.local', 'hitobito', { password: 'password',
+                                                           non_interactive: true,
+                                                           port: 22 })
+                       .and_return(session)
+      expect(session).to receive(:connect!)
+
+      subject.send(:connection)
+    end
+  end
+
+  context 'with private key' do
+    before { config.delete_field!(:password) }
+
+    it 'creates connection with private key' do
+      session = double
+
+      expect(::Net::SFTP).to receive(:start)
+                        .with('sftp.local', 'hitobito', { key_data: ['private key'],
+                                                          non_interactive: true,
+                                                          port: 22 })
+                       .and_return(session)
+      expect(session).to receive(:connect!)
+
+      subject.send(:connection)
+    end
+  end
+
+  context 'with private key and password' do
+    it 'creates connection with private key' do
+      session = double
+
+      expect(::Net::SFTP).to receive(:start)
+                         .with('sftp.local', 'hitobito', { key_data: ['private key'],
+                                                           non_interactive: true,
+                                                           port: 22 })
+                       .and_return(session)
+      expect(session).to receive(:connect!)
+
+      subject.send(:connection)
+    end
+  end
+
+end

spec/jobs/export/backup_mitglieder_export_job_spec.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas.
+
+require 'spec_helper'
+
+describe Export::BackupMitgliederExportJob do
+  subject(:job) { described_class.new }
+  let(:relevant_groups) { Group.where(type: [::Group::Sektion, ::Group::Ortsgruppe]) }
+
+  context 'rescheduling' do
+    it 'reschedules for tomorrow  at 5 minutes past midnight' do
+      job.perform
+      next_job = Delayed::Job.find_by("handler like '%BackupMitgliederExportJob%'")
+      expect(next_job.run_at).to eq Time.zone.tomorrow + 5.minutes
+    end
+  end
+
+  context 'perform' do
+    it 'only iterates over relevant groups' do
+      exporter = double
+      allow(exporter).to receive(:call)
+
+      relevant_groups.each do |group|
+        expect(BackupMitgliederExport).to receive(:new)
+          .with(group, an_instance_of(Sftp))
+          .and_return(exporter)
+
+      end
+
+      job.perform
+    end
+  end
+
+  context 'logging' do
+    let(:notifications) { Hash.new {|h, k| h[k] = [] } }
+
+    def subscribe
+      callback = lambda do |name, started, finished, unique_id, payload|
+        notifications[name] <<
+          OpenStruct.new(name: name, started: started, finished: finished, unique_id: unique_id, payload: payload)
+      end
+      ActiveSupport::Notifications.subscribed(callback, /\w+\.background_job/) do
+        yield
+      end
+    end
+
+    def run_job(payload_object)
+      payload_object.enqueue!.tap do |job_instance|
+        Delayed::Worker.new.run(job_instance)
+      end
+    end
+
+    it 'logs any type of error raised and continues' do
+      exporter = double
+      allow(exporter).to receive(:call)
+
+      error_group = relevant_groups.first
+
+      error = Sftp::ConnectionError.new('permission denied')
+
+      expect(BackupMitgliederExport).to receive(:new)
+                                    .with(error_group, an_instance_of(Sftp))
+                                    .and_raise(error)
+      expect(job).to receive(:error).with(job, error, group: error_group)
+
+      (relevant_groups - [error_group]).each do |group|
+        expect(BackupMitgliederExport).to receive(:new)
+          .with(group, an_instance_of(Sftp))
+          .and_return(exporter)
+      end
+
+      job = subscribe { run_job(subject) }
+
+      expect(notifications.keys).to match_array [
+        "job_started.background_job",
+        "job_finished.background_job"
+      ]
+
+      expect(notifications["job_started.background_job"]).to have(1).item
+      expect(notifications["job_finished.background_job"]).to have(1).item
+
+      started_attrs = notifications["job_started.background_job"].first[:payload]
+      expect(started_attrs).to match(
+        job_id: job.id,
+        job_name: described_class.name,
+        group_id: nil,
+        started_at: an_instance_of(ActiveSupport::TimeWithZone),
+        attempt: 0
+      )
+
+      finished_attrs = notifications["job_finished.background_job"].first[:payload]
+      expect(finished_attrs).to match(
+        job_id: job.id,
+        job_name: described_class.name,
+        group_id: nil,
+        finished_at: an_instance_of(ActiveSupport::TimeWithZone),
+        status: 'success',
+        payload: { errors: [[error_group.id, error]] },
+        attempt: 0
+      )
+    end
+  end
+end