examples/unit/archive_example.rb examples/unit/gpg_example.rb examples/unit/gzip_example.rb examples/unit/local_example.rb examples/unit/mysqldump_example.rb examples/unit/s3_example.rb lib/astrails/safe/backup.rb @@ -1,11 +1,6 @@ #!/usr/bin/env ruby - -require 'tempfile' require 'rubygems' -require 'fileutils' -require "aws/s3" -require 'yaml' #require 'ruby-debug' #$:.unshift File.join(File.dirname(__FILE__), "..", "lib") @@ -52,8 +47,7 @@ def main die "Created default #{$CONFIG_FILE_NAME}. Please edit and run again." end - load($CONFIG_FILE_NAME) end -main +main \ No newline at end of file bin/astrails-safe @@ -15,5 +15,4 @@ Micronaut.configure do |c| c.color_enabled = not_in_editor? c.filter_run :focused => true c.mock_with :rr -end - +end \ No newline at end of file examples/example_helper.rb @@ -120,7 +120,5 @@ describe Astrails::Safe::Config do } config.to_hash.should == expected - end -end - +end \ No newline at end of file examples/unit/config_example.rb @@ -1,4 +1,9 @@ +require "aws/s3" +require 'fileutils' + +require 'tempfile' require 'extensions/mktmpdir' + require 'astrails/safe/tmp_file' require 'astrails/safe/config/node' @@ -6,6 +11,10 @@ require 'astrails/safe/config/builder' require 'astrails/safe/stream' +require 'astrails/safe/backup' + +require 'astrails/safe/backup' + require 'astrails/safe/source' require 'astrails/safe/mysqldump' require 'astrails/safe/archive' @@ -23,16 +32,21 @@ module Astrails module Safe ROOT = File.join(File.dirname(__FILE__), "..", "..") - def timestamp - @timestamp ||= Time.now.strftime("%y%m%d-%H%M") - end - def safe(&block) config = Config::Node.new(&block) #config.dump - Astrails::Safe::Mysqldump.run(config[:mysqldump, :databases]) - Astrails::Safe::Archive.run(config[:tar, :archives]) + if databases = config[:mysqldump, :databases] + databases.each do |name, config| + Astrails::Safe::Mysqldump.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3) + end + end + + if archives = config[:tar, :archives] + archives.each do |name, config| + Astrails::Safe::Archive.new(name, config).backup.run(config, :gpg, :gzip, :local, :s3) + end + end Astrails::Safe::TmpFile.cleanup end lib/astrails/safe.rb @@ -16,9 +16,9 @@ module Astrails def tar_files raise RuntimeError, "missing files for tar" unless @config[:files] - [*@config[:files]] * " " + [*@config[:files]].map {|s| s.strip} * " " end end end -end +end \ No newline at end of file lib/astrails/safe/archive.rb @@ -57,4 +57,4 @@ module Astrails end end end -end +end \ No newline at end of file lib/astrails/safe/config/builder.rb @@ -60,8 +60,7 @@ module Astrails end end end - end end end -end +end \ No newline at end of file lib/astrails/safe/config/node.rb @@ -2,15 +2,14 @@ module Astrails module Safe class Gpg < Pipe - def compressed? - active? || @parent.compressed? - end - protected + def post_process + @backup.compressed = true + end + def pipe if key - rise RuntimeError, "can't use both gpg password and pubkey" if password "|gpg -e -r #{key}" elsif password "|gpg -c --passphrase-file #{gpg_password_file(password)}" @@ -18,19 +17,23 @@ module Astrails end def extension - ".gpg" if active? + ".gpg" end def active? + raise RuntimeError, "can't use both gpg password and pubkey" if key && password + password || key end + private + def password - @password ||= config[:gpg, :password] + @password ||= @config[:gpg, :password] end def key - @key ||= config[:gpg, :key] + @key ||= @config[:gpg, :key] end def gpg_password_file(pass) @@ -39,4 +42,4 @@ module Astrails end end end -end +end \ No newline at end of file lib/astrails/safe/gpg.rb @@ -2,24 +2,24 @@ module Astrails module Safe class Gzip < Pipe - def compressed? - true - end - protected + def post_process + @backup.compressed = true + end + def pipe - "|gzip" if active? + "|gzip" end def extension - ".gz" if active? + ".gz" end def active? - !@parent.compressed? + !@backup.compressed end end end -end +end \ No newline at end of file lib/astrails/safe/gzip.rb @@ -2,45 +2,37 @@ module Astrails module Safe class Local < Sink - def open(&block) - return @parent.open(&block) unless active? - run - File.open(path, &block) unless $DRY_RUN - end - protected def active? # S3 can't upload from pipe. it needs to know file size, so we must pass through :local - # will change once we add SSH sink + # will change once we add SSH/FTP sink true end def prefix - @prefix ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path in configuration"))) - end - - def command - "#{@parent.command} > #{path}" + @prefix ||= File.expand_path(expand(@config[:local, :path] || raise(RuntimeError, "missing :local/:path"))) end def save - puts "command: #{command}" if $_VERBOSE + puts "command: #{@backup.command}" if $_VERBOSE + unless $DRY_RUN FileUtils.mkdir_p(prefix) unless File.directory?(prefix) - system command + system "#{@backup.command}>#{@backup.path = path}" end + end def cleanup return unless keep = @config[:keep, :local] - base = File.basename(filename).split(".").first + base = File.basename(@backup.filename).split(".").first pattern = File.join(prefix, "#{base}*") puts "listing files #{pattern.inspect}" if $_VERBOSE files = Dir[pattern] . - select{|f| File.file?(f)} . + select{|f| File.file?(f) && File.size(f) > 0} . sort cleanup_with_limit(files, keep) do |f| @@ -48,7 +40,6 @@ module Astrails File.unlink(f) unless $DRY_RUN end end - end end end lib/astrails/safe/local.rb @@ -3,7 +3,7 @@ module Astrails class Mysqldump < Source def command - @commanbd ||= "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}" + "mysqldump --defaults-extra-file=#{mysql_password_file} #{@config[:options]} #{mysql_skip_tables} #{@id}" end def extension; '.sql'; end @@ -28,4 +28,4 @@ module Astrails end end -end +end \ No newline at end of file lib/astrails/safe/mysqldump.rb @@ -1,15 +1,13 @@ module Astrails module Safe class Pipe < Stream + def process + return unless active? - def command - "#{@parent.command}#{pipe}" + @backup.command << pipe + @backup.extension << extension + post_process end - - def filename - "#{@parent.filename}#{extension}" - end - end end end lib/astrails/safe/pipe.rb @@ -9,32 +9,30 @@ module Astrails end def prefix - @prefix ||= expand(config[:s3, :path] || expand(config[:local, :path] || ":kind/:id")) + @prefix ||= expand(config[:s3, :path] || config[:local, :path] || ":kind/:id") end def save + raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path + # needed in cleanup even on dry run AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL - file = @parent.open puts "Uploading #{bucket}:#{path}" if $_VERBOSE || $DRY_RUN unless $DRY_RUN || $LOCAL AWS::S3::Bucket.create(bucket) - AWS::S3::S3Object.store(path, file, bucket) + File.open(@backup.path) do |file| + AWS::S3::S3Object.store(path, file, bucket) + end puts "...done" if $_VERBOSE end - file.close if file - end def cleanup - return if $LOCAL return unless keep = @config[:keep, :s3] - bucket = @config[:s3, :bucket] - base = File.basename(filename).split(".").first puts "listing files in #{bucket}:#{prefix}/#{base}" @@ -52,15 +50,15 @@ module Astrails end def bucket - config[:s3, :bucket] + @config[:s3, :bucket] end def key - config[:s3, :key] + @config[:s3, :key] end def secret - config[:s3, :secret] + @config[:s3, :secret] end end lib/astrails/safe/s3.rb @@ -2,20 +2,18 @@ module Astrails module Safe class Sink < Stream - def run - if active? - save - cleanup - else - @parent.run - end + def process + return unless active? + + save + cleanup end protected # prefix is defined in subclass def path - @path ||= File.join(prefix, filename) + @path ||= File.join(prefix, @backup.filename) + @backup.extension end # call block on files to be removed (all except for the LAST 'limit' files @@ -26,8 +24,6 @@ module Astrails # TODO: validate here to_remove.each(&block) end - end end -end - +end \ No newline at end of file lib/astrails/safe/sink.rb @@ -2,27 +2,42 @@ module Astrails module Safe class Source < Stream + attr_accessor :id def initialize(id, config) - @id, @config = id, config + @id, @config = id.to_s, config + end + + def timestamp + Time.now.strftime("%y%m%d-%H%M") + end + + def kind + self.class.human_name end def filename - @filename ||= expand(":kind-:id.:timestamp#{extension}") + @filename ||= expand(":kind-:id.:timestamp") end - # process each config key as source (with full pipe) - def self.run(config) - unless config - puts "No configuration found for #{human_name}" - return - end - - config.each do |key, value| - stream = [Gpg, Gzip, Local, S3].inject(new(key, value)) do |res, klass| - klass.new(res) - end - stream.run - end + def backup + return @backup if @backup + @backup = Backup.new( + :id => @id, + :kind => kind, + :extension => extension, + :command => command, + :timestamp => timestamp + ) + # can't do this in the initializer hash above since + # filename() calls expand() which requires @backup + @backup.filename = filename + @backup + end + + protected + + def self.human_name + name.split('::').last.downcase end end lib/astrails/safe/source.rb @@ -2,44 +2,18 @@ module Astrails module Safe class Stream - def initialize(parent) - @parent = parent - end - - def id - @id ||= @parent.id - end - - def config - @config ||= @parent.config - end - - def filename - @parent.filename - end - - def compressed? - @parent && @parent.compressed? - end - - protected - - def self.human_name - name.split('::').last.downcase - end - - def kind - @parent ? @parent.kind : self.class.human_name + attr_accessor :config, :backup + def initialize(config, backup) + @config, @backup = config, backup end def expand(path) path . - gsub(/:kind\b/, kind) . - gsub(/:id\b/, id) . - gsub(/:timestamp\b/, timestamp) + gsub(/:kind\b/, @backup.kind.to_s) . + gsub(/:id\b/, @backup.id.to_s) . + gsub(/:timestamp\b/, @backup.timestamp) end end end -end - +end \ No newline at end of file lib/astrails/safe/stream.rb @@ -2,22 +2,26 @@ require 'tmpdir' module Astrails module Safe module TmpFile - @KEEP_FILES = [] - TMPDIR = Dir.mktmpdir + @keep_files = [] + + def self.tmproot + @tmproot ||= Dir.mktmpdir + end def self.cleanup - FileUtils.remove_entry_secure TMPDIR + FileUtils.remove_entry_secure tmproot + @tmproot = nil end def self.create(name) # create temp directory - file = Tempfile.new(name, TMPDIR) + file = Tempfile.new(name, tmproot) yield file file.close - @KEEP_FILES << file # so that it will not get gcollected and removed from filesystem until the end + @keep_files << file # so that it will not get gcollected and removed from filesystem until the end file.path end end lib/astrails/safe/tmp_file.rb @@ -6,13 +6,13 @@ Gem::Specification.new do |s| s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version= s.authors = ["Astrails Ltd."] - s.date = %q{2009-04-16} + s.date = %q{2009-04-25} s.default_executable = %q{astrails-safe} s.description = %q{Simple tool to backup MySQL databases and filesystem locally or to Amazon S3 (with optional encryption)} s.email = %q{we@astrails.com} s.executables = ["astrails-safe"] s.extra_rdoc_files = ["README.markdown", "LICENSE"] - s.files = ["README.markdown", "VERSION.yml", "bin/astrails-safe", "examples/example_helper.rb", "examples/unit", "examples/unit/config_example.rb", "examples/unit/stream_example.rb", "lib/astrails", "lib/astrails/safe", "lib/astrails/safe/archive.rb", "lib/astrails/safe/config", "lib/astrails/safe/config/builder.rb", "lib/astrails/safe/config/node.rb", "lib/astrails/safe/gpg.rb", "lib/astrails/safe/gzip.rb", "lib/astrails/safe/local.rb", "lib/astrails/safe/mysqldump.rb", "lib/astrails/safe/pipe.rb", "lib/astrails/safe/s3.rb", "lib/astrails/safe/sink.rb", "lib/astrails/safe/source.rb", "lib/astrails/safe/stream.rb", "lib/astrails/safe/tmp_file.rb", "lib/astrails/safe.rb", "lib/extensions", "lib/extensions/mktmpdir.rb", "templates/script.rb", "Rakefile", "LICENSE"] + s.files = ["README.markdown", "VERSION.yml", "bin/astrails-safe", "examples/example_helper.rb", "examples/unit", "examples/unit/archive_example.rb", "examples/unit/config_example.rb", "examples/unit/gpg_example.rb", "examples/unit/gzip_example.rb", "examples/unit/local_example.rb", "examples/unit/mysqldump_example.rb", "examples/unit/s3_example.rb", "lib/astrails", "lib/astrails/safe", "lib/astrails/safe/archive.rb", "lib/astrails/safe/backup.rb", "lib/astrails/safe/config", "lib/astrails/safe/config/builder.rb", "lib/astrails/safe/config/node.rb", "lib/astrails/safe/gpg.rb", "lib/astrails/safe/gzip.rb", "lib/astrails/safe/local.rb", "lib/astrails/safe/mysqldump.rb", "lib/astrails/safe/pipe.rb", "lib/astrails/safe/s3.rb", "lib/astrails/safe/sink.rb", "lib/astrails/safe/source.rb", "lib/astrails/safe/stream.rb", "lib/astrails/safe/tmp_file.rb", "lib/astrails/safe.rb", "lib/extensions", "lib/extensions/mktmpdir.rb", "templates/script.rb", "Rakefile", "LICENSE"] s.has_rdoc = true s.homepage = %q{http://github.com/astrails/safe} s.rdoc_options = ["--inline-source", "--charset=UTF-8"] safe.gemspec examples/unit/stream_example.rb 80cac2bf99ce4512c206ad77d54bf9a36684f97e Vitaly Kushner vitaly@astrails.com http://github.com/astrails/safe/commit/c2c572a2624d800275726f15451a162ed072bc47 c2c572a2624d800275726f15451a162ed072bc47 2009-04-24T14:25:23-07:00 2009-04-24T08:25:58-07:00 refactoring + testing 6df514eb1426d85ce7d6a6562ac4516811fe7018 Vitaly Kushner vitaly@astrails.com