@@ -7,6 +7,14 @@ aws_secret_access_key: --replace me with your AWS secret access key-- # Advanced customizations +# Use dynamic bucket path with erb +s3_bucket: <%= %> + +backup and compress all app files w/o log/* and tmp/* dirs +[config sample] + +added support of sqlite + # Turn on verbose output for debugging purposes: verbose: true @@ -26,8 +34,16 @@ static_paths: "/u/apps/foo/current/public/static" # The default dump base path is 'tmp/backups'. This can be changed with: dump_base_path: /tmp -# To disable the tar/gzipping of the DB backup: -disable_tar_gzip: true +# To use tar-gzipping or zipping for compression. Use the values tar-gzip or zip here. Default is tar-gzip. +compressor: tar-gzip + +# To disable compression of the DB backup: +disable_compression: true + +# Password-protect zip files. The password will be omitted if empty. +# This is a weak protection +# Read the man page of zip for clearance of security +zip_password: <your-password-here> # To turn on using 'nice' to keep backup CPU processing to a minimum: enable_nice: true config/backup_fu.yml.advanced_example @@ -1,6 +1,8 @@ require 'yaml' require 'active_support' -require 'aws/s3' +require 'mime/types' +require 'right_aws' +require 'erb' class BackupFuConfigError < StandardError; end class S3ConnectError < StandardError; end @@ -10,8 +12,12 @@ class BackupFu def initialize db_conf = YAML.load_file(File.join(RAILS_ROOT, 'config', 'database.yml')) @db_conf = db_conf[RAILS_ENV].symbolize_keys - fu_conf = YAML.load_file(File.join(RAILS_ROOT, 'config', 'backup_fu.yml')) - @fu_conf = fu_conf[RAILS_ENV].symbolize_keys + + raw_config = File.read(File.join(RAILS_ROOT, 'config', 'backup_fu.yml')) + erb_config = ERB.new(raw_config).result + fu_conf = YAML.load(erb_config) + @fu_conf = fu_conf[RAILS_ENV].symbolize_keys + @fu_conf[:mysqldump_options] ||= '--complete-insert --skip-extended-insert' @verbose = !@fu_conf[:verbose].nil? @timestamp = datetime_formatted @@ -31,42 +37,30 @@ class BackupFu if @db_conf.has_key?(:password) && !@db_conf[:password].blank? password = "--password=#{@db_conf[:password]}" end + full_dump_path = File.join(dump_base_path, db_filename) case @db_conf[:adapter] - when 'postgresql' - cmd = niceify "PGPASSWORD=#{password} #{dump_path} --user=#{@db_conf[:username]} --host=#{host} --port=#{port} #{@db_conf[:database]} > #{full_dump_path}" - when 'mysql' - cmd = niceify "#{dump_path} #{@fu_conf[:mysqldump_options]} #{host} #{port} --user=#{@db_conf[:username]} #{password} #{@db_conf[:database]} > #{full_dump_path}" + when 'postgresql' + cmd = niceify "PGPASSWORD=#{password} #{dump_path} --user=#{@db_conf[:username]} --host=#{host} --port=#{port} #{@db_conf[:database]} > #{full_dump_path}" + when 'mysql' + cmd = niceify "#{dump_path} #{@fu_conf[:mysqldump_options]} #{host} #{port} --user=#{@db_conf[:username]} #{password} #{@db_conf[:database]} > #{full_dump_path}" end puts cmd if @verbose `#{cmd}` - if !@fu_conf[:disable_tar_gzip] - - tar_path = File.join(dump_base_path, db_filename_tarred) - - # TAR it up - cmd = niceify "tar -cf #{tar_path} -C #{dump_base_path} #{db_filename}" - puts "\nTar: #{cmd}\n" if @verbose - `#{cmd}` - - # GZip it up - cmd = niceify "gzip -f #{tar_path}" - puts "\nGzip: #{cmd}" if @verbose - `#{cmd}` + if !@fu_conf[:disable_compression] + compress_db(dump_base_path, db_filename) + File.unlink full_dump_path end - end - + def backup dump - establish_s3_connection file = final_db_dump_path() puts "\nBacking up to S3: #{file}\n" if @verbose - AWS::S3::S3Object.store(File.basename(file), open(file), @fu_conf[:s3_bucket], :access => :private) - + store_file(file) end ## Static-file Dump/Backup methods @@ -75,47 +69,19 @@ class BackupFu if !@fu_conf[:static_paths] raise BackupFuConfigError, 'No static paths are defined in config/backup_fu.yml. See README.' end - @paths = @fu_conf[:static_paths].split(' ') - path_num = 0 - @paths.each do |p| - if p.first != '/' - # Make into an Absolute path: - p = File.join(RAILS_ROOT, p) - end - - puts "Static Path: #{p}" if @verbose - if path_num == 0 - tar_switch = 'c' # for create - else - tar_switch = 'r' # for append - end - - # TAR - cmd = niceify "tar -#{tar_switch}f #{static_tar_path} #{p}" - puts "\nTar: #{cmd}\n" if @verbose - `#{cmd}` - - path_num += 1 - end - - # GZIP - cmd = niceify "gzip -f #{static_tar_path}" - puts "\nGzip: #{cmd}" if @verbose - `#{cmd}` - + paths = @fu_conf[:static_paths].split(' ') + compress_static(paths) end def backup_static dump_static - establish_s3_connection file = final_static_dump_path() puts "\nBacking up Static files to S3: #{file}\n" if @verbose - AWS::S3::S3Object.store(File.basename(file), open(file), @fu_conf[:s3_bucket], :access => :private) - + store_file(file) end - + def cleanup count = @fu_conf[:keep_backups].to_i backups = Dir.glob("#{dump_base_path}/*.{sql}") @@ -123,27 +89,39 @@ class BackupFu puts "no old backups to cleanup" else puts "keeping #{count} of #{backups.length} backups" - + files_to_remove = backups - backups.last(count) - files_to_remove = files_to_remove.concat(Dir.glob("#{dump_base_path}/*.{gz}")[0, files_to_remove.length]) unless @fu_conf[:disable_tar_gzip] - + + if(!@fu_conf[:disable_compression]) + if(@fu_conf[:compressor] == 'zip') + files_to_remove = files_to_remove.concat(Dir.glob("#{dump_base_path}/*.{zip}")[0, files_to_remove.length]) + else + files_to_remove = files_to_remove.concat(Dir.glob("#{dump_base_path}/*.{gz}")[0, files_to_remove.length]) + end + end + files_to_remove.each do |f| File.delete(f) end - + end end private - def establish_s3_connection - unless AWS::S3::Base.connected? - AWS::S3::Base.establish_connection!( - :access_key_id => @fu_conf[:aws_access_key_id], - :secret_access_key => @fu_conf[:aws_secret_access_key] - ) - end - raise S3ConnectError, "\nERROR: Connection to Amazon S3 failed." unless AWS::S3::Base.connected? + def s3 + @s3 ||= RightAws::S3.new(@fu_conf[:aws_access_key_id], + @fu_conf[:aws_secret_access_key]) + end + + def s3_bucket + @s3_bucket ||= s3.bucket(@fu_conf[:s3_bucket], true, 'private') + end + + def store_file(file) + key = s3_bucket.key(File.basename(file)) + key.data = open(file) + key.put(nil, 'private') end def check_conf @@ -175,30 +153,46 @@ class BackupFu def db_filename "#{@fu_conf[:app_name]}_#{ @timestamp }_db.sql" end - - def db_filename_tarred - db_filename.gsub('.sql', '.tar') + + def db_filename_compressed + if(@fu_conf[:compressor] == 'zip') + db_filename.gsub('.sql', '.zip') + else + db_filename.gsub('.sql', '.tar') + end end - + def final_db_dump_path - if @fu_conf[:disable_tar_gzip] + if(@fu_conf[:disable_compression]) filename = db_filename else - filename = db_filename.gsub('.sql', '.tar.gz') + if(@fu_conf[:compressor] == 'zip') + filename = db_filename.gsub('.sql', '.zip') + else + filename = db_filename.gsub('.sql', '.tar.gz') + end end File.join(dump_base_path, filename) end - - def static_tar_path - f = "#{@fu_conf[:app_name]}_#{ @timestamp }_static.tar" + + def static_compressed_path + if(@fu_conf[:compressor] == 'zip') + f = "#{@fu_conf[:app_name]}_#{ @timestamp }_static.zip" + else + f = "#{@fu_conf[:app_name]}_#{ @timestamp }_static.tar" + end File.join(dump_base_path, f) end - + def final_static_dump_path - f = "#{@fu_conf[:app_name]}_#{ @timestamp }_static.tar.gz" + if(@fu_conf[:compressor] == 'zip') + f = "#{@fu_conf[:app_name]}_#{ @timestamp }_static.zip" + else + f = "#{@fu_conf[:app_name]}_#{ @timestamp }_static.tar.gz" + end File.join(dump_base_path, f) end - + def create_dirs ensure_directory_exists(dump_base_path) end @@ -219,4 +213,80 @@ class BackupFu Time.now.strftime("%Y-%m-%d") + "_#{ Time.now.tv_sec }" end -end \ No newline at end of file + def compress_db(dump_base_path, db_filename) + compressed_path = File.join(dump_base_path, db_filename_compressed) + + if(@fu_conf[:compressor] == 'zip') + cmd = niceify "zip #{zip_switch} #{compressed_path} #{dump_base_path}/#{db_filename}" + puts "\nZip: #{cmd}\n" if @verbose + `#{cmd}` + else + + # TAR it up + cmd = niceify "tar -cf #{compressed_path} -C #{dump_base_path} #{db_filename}" + puts "\nTar: #{cmd}\n" if @verbose + `#{cmd}` + + # GZip it up + cmd = niceify "gzip -f #{compressed_path}" + puts "\nGzip: #{cmd}" if @verbose + `#{cmd}` + end + end + + def compress_static(paths) + path_num = 0 + paths.each do |p| + if p.first != '/' + # Make into an Absolute path: + p = File.join(RAILS_ROOT, p) + end + + puts "Static Path: #{p}" if @verbose + + if @fu_conf[:compressor] == 'zip' + cmd = niceify "zip -r #{zip_switch} #{static_compressed_path} #{p}" + puts "\nZip: #{cmd}\n" if @verbose + `#{cmd}` + else + if path_num == 0 + tar_switch = 'c' # for create + else + tar_switch = 'r' # for append + end + + # TAR + cmd = niceify "tar -#{tar_switch}f #{static_compressed_path} #{p}" + puts "\nTar: #{cmd}\n" if @verbose + `#{cmd}` + + path_num += 1 + + # GZIP + cmd = niceify "gzip -f #{static_compressed_path}" + puts "\nGzip: #{cmd}" if @verbose + `#{cmd}` + end + end + end + + def zip_switch + if(@fu_conf[:zip_password] && !@fu_conf[:zip_password].blank?) + "-P #{@fu_conf[:zip_password]}" + else + '' + end + end + + def skips + return '' unless @fu_conf[:skips] + + raise BackupFuConfigError, 'skip option is not array or string' unless @fu_conf[:skips].kind_of?(Array) || @fu_conf[:skips].kind_of?(String) + + if @fu_conf[:skips].kind_of?(Array) + @fu_conf[:skips].collect{|skip| " --exclude=#{skip} " }.join + else + @fu_conf[:skips] + end + end +end lib/backup_fu.rb @@ -51,7 +51,7 @@ namespace :backup_fu do namespace :static do - desc "Tars and gzips static application files locally. Does *not* upload to S3." + desc "Zips or Tars and gzips static application files locally. Does *not* upload to S3." task :dump do b = BackupFu.new b.dump_static tasks/backup_fu_tasks.rake 7718d209a3060a8314b722baa78bb8b8388f8434 b50d0638ce4cf6abcd5ca0a4130e2ae83d3cd6fd Bill Kirtley bill@virosity.com http://github.com/gravelpup/backup_fu/commit/8abf0605b2084704e869eecf29e4f541d46d7d13 8abf0605b2084704e869eecf29e4f541d46d7d13 2009-01-28T12:52:15-08:00 2009-01-28T12:52:15-08:00 Merge bigcurl support for zip and right_aws b48b8b8b36a84fa79e2228d932b36ebbcc359fae Bill Kirtley bill@virosity.com