REFACTOR: Restoring of backups and migration of uploads to S3

.github/workflows/ci.yml

@@ -1,13 +1,13 @@
 name: CI
-   
+
 on:
   push:
     branches:
       - master
   pull_request:
     branches-ignore:
       - 'tests-passed'
-    
+
 jobs:
   build:
     name: "${{ matrix.target }}-${{ matrix.build_types }}"
@@ -38,7 +38,7 @@ jobs:
     services:
       postgres:
         image: postgres:${{ matrix.postgres }}
-        ports: 
+        ports:
           - 5432:5432
         env:
           POSTGRES_USER: discourse
@@ -88,14 +88,14 @@ jobs:
           key: ${{ runner.os }}-gem-${{ hashFiles('**/Gemfile.lock') }}
           restore-keys: |
             ${{ runner.os }}-gem-
-      
+
       - name: Setup gems
         run: bundle install --without development --deployment --jobs 4 --retry 3
 
       - name: Get yarn cache directory
         id: yarn-cache-dir
         run: echo "::set-output name=dir::$(yarn cache dir)"
-    
+
       - name: Yarn cache
         uses: actions/cache@v1
         id: yarn-cache
@@ -113,7 +113,7 @@ jobs:
         run: bin/rake plugin:install_all_official
 
       - name: Create database
-        if: env.BUILD_TYPE != 'LINT'    
+        if: env.BUILD_TYPE != 'LINT'
         run: bin/rake db:create && bin/rake db:migrate
 
       - name: Create parallel databases
@@ -123,7 +123,7 @@ jobs:
       - name: Rubocop
         if: env.BUILD_TYPE == 'LINT'
         run: bundle exec rubocop .
-      
+
       - name: ESLint
         if: env.BUILD_TYPE == 'LINT'
         run: yarn eslint app/assets/javascripts test/javascripts && yarn eslint --ext .es6 app/assets/javascripts test/javascripts plugins
@@ -133,7 +133,7 @@ jobs:
         run: |
           yarn prettier -v
           yarn prettier --list-different "app/assets/stylesheets/**/*.scss" "app/assets/javascripts/**/*.es6" "test/javascripts/**/*.es6" "plugins/**/*.scss" "plugins/**/*.es6"
-        
+
       - name: Core RSpec
         if: env.BUILD_TYPE == 'BACKEND' && env.TARGET == 'CORE'
         run: bin/turbo_rspec && bin/rake plugin:spec
@@ -146,12 +146,12 @@ jobs:
         if: env.BUILD_TYPE == 'FRONTEND' && env.TARGET == 'CORE'
         run: bundle exec rake qunit:test['1200000']
         timeout-minutes: 30
-      
+
       - name: Wizard QUnit
         if: env.BUILD_TYPE == 'FRONTEND' && env.TARGET == 'CORE'
         run: bundle exec rake qunit:test['1200000','/wizard/qunit']
         timeout-minutes: 30
-          
+
       - name: Plugin QUnit # Tests core plugins in TARGET=CORE, and all plugins in TARGET=PLUGINS
         if: env.BUILD_TYPE == 'FRONTEND'
         run: bundle exec rake plugin:qunit

.gitignore

@@ -32,6 +32,7 @@ config/discourse.conf
 # Ignore the default SQLite database and db dumps
 *.sql
 *.sql.gz
+!/spec/fixtures/**/*.sql
 /db/*.sqlite3
 /db/structure.sql
 /db/schema.rb

lib/backup_restore.rb

@@ -4,10 +4,8 @@ module BackupRestore
 
   class OperationRunningError < RuntimeError; end
 
-  VERSION_PREFIX = "v".freeze
-  DUMP_FILE = "dump.sql.gz".freeze
-  OLD_DUMP_FILE = "dump.sql".freeze
-  METADATA_FILE = "meta.json"
+  VERSION_PREFIX = "v"
+  DUMP_FILE = "dump.sql.gz"
   LOGS_CHANNEL = "/admin/backups/logs"
 
   def self.backup!(user_id, opts = {})
@@ -19,7 +17,16 @@ def self.backup!(user_id, opts = {})
   end
 
   def self.restore!(user_id, opts = {})
-    start! BackupRestore::Restorer.new(user_id, opts)
+    restorer = BackupRestore::Restorer.new(
+      user_id: user_id,
+      filename: opts[:filename],
+      factory: BackupRestore::Factory.new(
+        user_id: user_id,
+        client_id: opts[:client_id]
+      )
+    )
+
+    start! restorer
   end
 
   def self.rollback!
@@ -75,16 +82,18 @@ def self.current_version
   end
 
   def self.move_tables_between_schemas(source, destination)
-    DB.exec(move_tables_between_schemas_sql(source, destination))
+    ActiveRecord::Base.transaction do
+      DB.exec(move_tables_between_schemas_sql(source, destination))
+    end
   end
 
   def self.move_tables_between_schemas_sql(source, destination)
-    <<-SQL
+    <<~SQL
       DO $$DECLARE row record;
       BEGIN
         -- create <destination> schema if it does not exists already
         -- NOTE: DROP & CREATE SCHEMA is easier, but we don't want to drop the public schema
-        -- ortherwise extensions (like hstore & pg_trgm) won't work anymore...
+        -- otherwise extensions (like hstore & pg_trgm) won't work anymore...
         CREATE SCHEMA IF NOT EXISTS #{destination};
         -- move all <source> tables to <destination> schema
         FOR row IN SELECT tablename FROM pg_tables WHERE schemaname = '#{source}'
@@ -108,11 +117,17 @@ def self.database_configuration
     config = ActiveRecord::Base.connection_pool.spec.config
     config = config.with_indifferent_access
 
+    # credentials for PostgreSQL in CI environment
+    if Rails.env.test?
+      username = ENV["PGUSER"]
+      password = ENV["PGPASSWORD"]
+    end
+
     DatabaseConfiguration.new(
       config["backup_host"] || config["host"],
       config["backup_port"] || config["port"],
-      config["username"] || ENV["USER"] || "postgres",
-      config["password"],
+      config["username"] || username || ENV["USER"] || "postgres",
+      config["password"] || password,
       config["database"]
     )
   end

lib/backup_restore/backup_file_handler.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module BackupRestore
+  class BackupFileHandler
+    OLD_DUMP_FILENAME = "dump.sql"
+
+    delegate :log, to: :@logger, private: true
+
+    def initialize(logger, filename, current_db, root_tmp_directory = Rails.root)
+      @logger = logger
+      @filename = filename
+      @current_db = current_db
+      @root_tmp_directory = root_tmp_directory
+      @is_archive = !(@filename =~ /\.sql\.gz$/)
+    end
+
+    def decompress
+      create_tmp_directory
+      @archive_path = File.join(@tmp_directory, @filename)
+
+      copy_archive_to_tmp_directory
+      decompress_archive
+      extract_db_dump
+
+      [@tmp_directory, @db_dump_path]
+    end
+
+    def clean_up
+      return if @tmp_directory.blank?
+
+      log "Removing tmp '#{@tmp_directory}' directory..."
+      FileUtils.rm_rf(@tmp_directory) if Dir[@tmp_directory].present?
+    rescue => ex
+      log "Something went wrong while removing the following tmp directory: #{@tmp_directory}", ex
+    end
+
+    protected
+
+    def create_tmp_directory
+      timestamp = Time.zone.now.strftime("%Y-%m-%d-%H%M%S")
+      @tmp_directory = File.join(@root_tmp_directory, "tmp", "restores", @current_db, timestamp)
+      ensure_directory_exists(@tmp_directory)
+    end
+
+    def ensure_directory_exists(directory)
+      log "Making sure #{directory} exists..."
+      FileUtils.mkdir_p(directory)
+    end
+
+    def copy_archive_to_tmp_directory
+      store = BackupRestore::BackupStore.create
+
+      if store.remote?
+        log "Downloading archive to tmp directory..."
+        failure_message = "Failed to download archive to tmp directory."
+      else
+        log "Copying archive to tmp directory..."
+        failure_message = "Failed to copy archive to tmp directory."
+      end
+
+      store.download_file(@filename, @archive_path, failure_message)
+    end
+
+    def decompress_archive
+      return if !@is_archive
+
+      log "Unzipping archive, this may take a while..."
+      pipeline = Compression::Pipeline.new([Compression::Tar.new, Compression::Gzip.new])
+      unzipped_path = pipeline.decompress(@tmp_directory, @archive_path, available_size)
+      pipeline.strip_directory(unzipped_path, @tmp_directory)
+    end
+
+    def extract_db_dump
+      @db_dump_path =
+        if @is_archive
+          # for compatibility with backups from Discourse v1.5 and below
+          old_dump_path = File.join(@tmp_directory, OLD_DUMP_FILENAME)
+          File.exists?(old_dump_path) ? old_dump_path : File.join(@tmp_directory, BackupRestore::DUMP_FILE)
+        else
+          File.join(@tmp_directory, @filename)
+        end
+
+      if File.extname(@db_dump_path) == '.gz'
+        log "Extracting dump file..."
+        Compression::Gzip.new.decompress(@tmp_directory, @db_dump_path, available_size)
+        @db_dump_path.delete_suffix!('.gz')
+      end
+
+      @db_dump_path
+    end
+
+    def available_size
+      SiteSetting.decompressed_backup_max_file_size_mb
+    end
+  end
+end

lib/backup_restore/backup_store.rb

@@ -3,8 +3,8 @@
 module BackupRestore
   # @abstract
   class BackupStore
-    class BackupFileExists < RuntimeError; end
-    class StorageError < RuntimeError; end
+    BackupFileExists = Class.new(RuntimeError)
+    StorageError = Class.new(RuntimeError)
 
     # @return [BackupStore]
     def self.create(opts = {})