backup/restore rake task

CHANGELOG

@@ -10,6 +10,7 @@ v7.11.0
   - Remove projects IDs from dashboard
   - UI fix: Remove page headers from the admin area
   - Improve Email templates
+  - Add backup/restore utility
 
 v7.10.1
   - Fix failing migration when update to 7.10 from 7.8 and older versions

Gemfile

@@ -85,6 +85,9 @@ gem 'nprogress-rails'
 # Soft deletion
 gem "paranoia", "~> 2.0"
 
+# Colored output to console
+gem "colored"
+
 
 group :development do
   gem 'brakeman', require: false

Gemfile.lock

@@ -81,6 +81,7 @@ GEM
       coffee-script-source
       execjs
     coffee-script-source (1.6.3)
+    colored (1.2)
     columnize (0.9.0)
     connection_pool (1.2.0)
     coveralls (0.7.0)
@@ -380,6 +381,7 @@ DEPENDENCIES
   byebug
   capybara
   coffee-rails (~> 4.0.0)
+  colored
   coveralls
   database_cleaner
   default_value_for (~> 3.0.0)

config/initializers/1_settings.rb

@@ -51,3 +51,15 @@ def build_gitlab_ci_url
 Settings.gravatar['enabled']     = true if Settings.gravatar['enabled'].nil?
 Settings.gravatar['plain_url'] ||= 'http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=mm'
 Settings.gravatar['ssl_url']   ||= 'https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=mm'
+
+#
+# Backup
+#
+Settings['backup'] ||= Settingslogic.new({})
+Settings.backup['keep_time']  ||= 0
+Settings.backup['path']         = File.expand_path(Settings.backup['path'] || "tmp/backups/", Rails.root)
+Settings.backup['upload'] ||= Settingslogic.new({ 'remote_directory' => nil, 'connection' => nil })
+# Convert upload connection settings to use symbol keys, to make Fog happy
+if Settings.backup['upload']['connection']
+  Settings.backup['upload']['connection'] = Hash[Settings.backup['upload']['connection'].map { |k, v| [k.to_sym, v] }]
+end

lib/backup/database.rb

@@ -0,0 +1,92 @@
+require 'yaml'
+
+module Backup
+  class Database
+    attr_reader :config, :db_dir
+
+    def initialize
+      @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env]
+      @db_dir = File.join(GitlabCi.config.backup.path, 'db')
+      FileUtils.mkdir_p(@db_dir) unless Dir.exists?(@db_dir)
+    end
+
+    def dump
+      success = case config["adapter"]
+      when /^mysql/ then
+        $progress.print "Dumping MySQL database #{config['database']} ... "
+        system('mysqldump', *mysql_args, config['database'], out: db_file_name)
+      when "postgresql" then
+        $progress.print "Dumping PostgreSQL database #{config['database']} ... "
+        pg_env
+        system('pg_dump', config['database'], out: db_file_name)
+      end
+      report_success(success)
+      abort 'Backup failed' unless success
+    end
+
+    def restore
+      success = case config["adapter"]
+      when /^mysql/ then
+        $progress.print "Restoring MySQL database #{config['database']} ... "
+        system('mysql', *mysql_args, config['database'], in: db_file_name)
+      when "postgresql" then
+        $progress.print "Restoring PostgreSQL database #{config['database']} ... "
+        # Drop all tables because PostgreSQL DB dumps do not contain DROP TABLE
+        # statements like MySQL.
+        drop_all_tables
+        drop_all_postgres_sequences
+        pg_env
+        system('psql', config['database'], '-f', db_file_name)
+      end
+      report_success(success)
+      abort 'Restore failed' unless success
+    end
+
+    protected
+
+    def db_file_name
+      File.join(db_dir, 'database.sql')
+    end
+
+    def mysql_args
+      args = {
+        'host'      => '--host',
+        'port'      => '--port',
+        'socket'    => '--socket',
+        'username'  => '--user',
+        'encoding'  => '--default-character-set',
+        'password'  => '--password'
+      }
+      args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact
+    end
+
+    def pg_env
+      ENV['PGUSER']     = config["username"] if config["username"]
+      ENV['PGHOST']     = config["host"] if config["host"]
+      ENV['PGPORT']     = config["port"].to_s if config["port"]
+      ENV['PGPASSWORD'] = config["password"].to_s if config["password"]
+    end
+
+    def report_success(success)
+      if success
+        $progress.puts '[DONE]'.green
+      else
+        $progress.puts '[FAILED]'.red
+      end
+    end
+
+    def drop_all_tables
+      connection = ActiveRecord::Base.connection
+      connection.tables.each do |table|
+        connection.drop_table(table)
+      end
+    end
+
+    def drop_all_postgres_sequences
+      connection = ActiveRecord::Base.connection
+      connection.execute("SELECT c.relname FROM pg_class c WHERE c.relkind = 'S';").each do |sequence|
+        connection.execute("DROP SEQUENCE #{sequence['relname']}")
+      end
+    end
+  end
+end

lib/backup/manager.rb

@@ -0,0 +1,155 @@
+module Backup
+  class Manager
+    def pack
+      # saving additional informations
+      s = {}
+      s[:db_version]         = "#{ActiveRecord::Migrator.current_version}"
+      s[:backup_created_at]  = Time.now
+      s[:gitlab_version]     = GitlabCi::VERSION
+      s[:tar_version]        = tar_version
+      tar_file = "#{s[:backup_created_at].to_i}_gitlab_ci_backup.tar"
+
+      Dir.chdir(GitlabCi.config.backup.path) do
+        File.open("#{GitlabCi.config.backup.path}/backup_information.yml",
+                  "w+") do |file|
+          file << s.to_yaml.gsub(/^---\n/,'')
+        end
+
+        FileUtils.chmod(0700, "db")
+
+        # create archive
+        $progress.print "Creating backup archive: #{tar_file} ... "
+        orig_umask = File.umask(0077)
+        if Kernel.system('tar', '-cf', tar_file, *backup_contents)
+          $progress.puts "done".green
+        else
+          puts "creating archive #{tar_file} failed".red
+          abort 'Backup failed'
+        end
+        File.umask(orig_umask)
+
+        upload(tar_file)
+      end
+    end
+
+    def upload(tar_file)
+      remote_directory = GitlabCi.config.backup.upload.remote_directory
+      $progress.print "Uploading backup archive to remote storage #{remote_directory} ... "
+
+      connection_settings = GitlabCi.config.backup.upload.connection
+      if connection_settings.blank?
+        $progress.puts "skipped".yellow
+        return
+      end
+
+      connection = ::Fog::Storage.new(connection_settings)
+      directory = connection.directories.get(remote_directory)
+
+      if directory.files.create(key: tar_file, body: File.open(tar_file), public: false)
+        $progress.puts "done".green
+      else
+        puts "uploading backup to #{remote_directory} failed".red
+        abort 'Backup failed'
+      end
+    end
+
+    def cleanup
+      $progress.print "Deleting tmp directories ... "
+
+      backup_contents.each do |dir|
+        next unless File.exist?(File.join(GitlabCi.config.backup.path, dir))
+
+        if FileUtils.rm_rf(File.join(GitlabCi.config.backup.path, dir))
+          $progress.puts "done".green
+        else
+          puts "deleting tmp directory '#{dir}' failed".red
+          abort 'Backup failed'
+        end
+      end
+    end
+
+    def remove_old
+      # delete backups
+      $progress.print "Deleting old backups ... "
+      keep_time = GitlabCi.config.backup.keep_time.to_i
+
+      if keep_time > 0
+        removed = 0
+
+        Dir.chdir(GitlabCi.config.backup.path) do
+          file_list = Dir.glob('*_gitlab_ci_backup.tar')
+          file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_ci_backup.tar/ }
+          file_list.sort.each do |timestamp|
+            if Time.at(timestamp) < (Time.now - keep_time)
+              if Kernel.system(*%W(rm #{timestamp}_gitlab_ci_backup.tar))
+                removed += 1
+              end
+            end
+          end
+        end
+
+        $progress.puts "done. (#{removed} removed)".green
+      else
+        $progress.puts "skipping".yellow
+      end
+    end
+
+    def unpack
+      Dir.chdir(GitlabCi.config.backup.path)
+
+      # check for existing backups in the backup dir
+      file_list = Dir.glob("*_gitlab_ci_backup.tar").each.map { |f| f.split(/_/).first.to_i }
+      puts "no backups found" if file_list.count == 0
+
+      if file_list.count > 1 && ENV["BACKUP"].nil?
+        puts "Found more than one backup, please specify which one you want to restore:"
+        puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup"
+        exit 1
+      end
+
+      tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_ci_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_ci_backup.tar")
+
+      unless File.exists?(tar_file)
+        puts "The specified backup doesn't exist!"
+        exit 1
+      end
+
+      $progress.print "Unpacking backup ... "
+
+      unless Kernel.system(*%W(tar -xf #{tar_file}))
+        puts "unpacking backup failed".red
+        exit 1
+      else
+        $progress.puts "done".green
+      end
+
+      ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0
+
+      # restoring mismatching backups can lead to unexpected problems
+      if settings[:gitlab_version] != GitlabCi::VERSION
+        puts "GitLab CI version mismatch:".red
+        puts "  Your current GitLab CI version (#{GitlabCi::VERSION}) differs from the GitLab CI version in the backup!".red
+        puts "  Please switch to the following version and try again:".red
+        puts "  version: #{settings[:gitlab_version]}".red
+        puts
+        puts "Hint: git checkout v#{settings[:gitlab_version]}"
+        exit 1
+      end
+    end
+
+    def tar_version
+      tar_version = `tar --version`
+      tar_version.force_encoding('locale').split("\n").first
+    end
+
+    private
+
+    def backup_contents
+      ["db", "backup_information.yml"]
+    end
+
+    def settings
+      @settings ||= YAML.load_file("backup_information.yml")
+    end
+  end
+end

lib/tasks/backup.rake

@@ -0,0 +1,43 @@
+namespace :backup do
+
+  desc "GITLAB | Create a backup of the GitLab CI database"
+  task create: :environment do
+    configure_cron_mode
+
+    $progress.puts "Dumping database ... ".blue
+
+    Backup::Database.new.dump
+    $progress.puts "done".green
+
+    backup = Backup::Manager.new
+    backup.pack
+    backup.cleanup
+    backup.remove_old
+  end
+
+  desc "GITLAB | Restore a previously created backup"
+  task restore: :environment do
+    configure_cron_mode
+
+    backup = Backup::Manager.new
+    backup.unpack
+
+    $progress.puts "Restoring database ... ".blue
+    Backup::Database.new.restore
+    $progress.puts "done".green
+
+    backup.cleanup
+  end
+
+  def configure_cron_mode
+    if ENV['CRON']
+      # We need an object we can say 'puts' and 'print' to; let's use a
+      # StringIO.
+      require 'stringio'
+      $progress = StringIO.new
+    else
+      $progress = $stdout
+    end
+  end
+end
+