Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Comparing changes

Choose two branches to see what's changed or to start a new pull request. If you need to, you can also compare across forks.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also compare across forks.
...
Checking mergeability… Don't worry, you can still create the pull request.
  • 5 commits
  • 8 files changed
  • 0 commit comments
  • 1 contributor
View
6 astrails-safe.gemspec
@@ -30,10 +30,8 @@ Remote storage is supported on Amazon S3, Rackspace Cloud Files, or just plain S
s.add_development_dependency 'rspec', '~> 1.3.2'
s.add_development_dependency 'rr', '~> 1.0.4'
- s.add_runtime_dependency 'aws-s3', '~> 0.6.2'
+ s.add_runtime_dependency 'aws-sdk', '~> 1.2.3'
s.add_runtime_dependency 'cloudfiles', '~> 1.4.7'
s.add_runtime_dependency 'net-sftp', '~> 2.0.4'
- s.add_runtime_dependency 'toadhopper', '~> 2.0'
-
+ s.add_runtime_dependency 'toadhopper', '~> 2.0'
end
-
View
4 lib/astrails/safe.rb
@@ -1,4 +1,4 @@
-require "aws/s3"
+require "aws-sdk"
require "cloudfiles"
require 'net/sftp'
require 'fileutils'
@@ -22,6 +22,7 @@
require 'astrails/safe/pgdump'
require 'astrails/safe/archive'
require 'astrails/safe/svndump'
+require 'astrails/safe/mongodump'
require 'astrails/safe/pipe'
require 'astrails/safe/gpg'
@@ -45,6 +46,7 @@ def safe(&block)
begin
[[Mysqldump, [:mysqldump, :databases]],
[Pgdump, [:pgdump, :databases]],
+ [Mongodump, [:mongodump, :databases]],
[Archive, [:tar, :archives]],
[Svndump, [:svndump, :repos]]
].each do |klass, path|
View
2  lib/astrails/safe/config/builder.rb
@@ -4,7 +4,7 @@ module Config
class Builder
COLLECTIONS = %w/database archive repo/
ITEMS = %w/s3 cloudfiles key secret bucket api_key container service_net path gpg password keep local mysqldump pgdump command options
- user host port socket skip_tables tar files exclude filename svndump repo_path sftp airbrake/
+ user host port socket skip_tables tar files exclude filename svndump repo_path sftp airbrake mongodump/
NAMES = COLLECTIONS + ITEMS
def initialize(node)
@node = node
View
23 lib/astrails/safe/mongodump.rb
@@ -0,0 +1,23 @@
+module Astrails
+ module Safe
+ class Mongodump < Source
+
+ def command
+ opts = []
+ opts << "--host #{@config[:host]}" if @config[:host]
+ opts << "-u #{@config[:user]}" if @config[:user]
+ opts << "-p #{@config[:password]}" if @config[:password]
+ opts << "--out #{output_directory}"
+
+ "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db #{@id} #{opts.join(" ")} && cd #{output_directory} && tar cf - ."
+ end
+
+ def extension; '.tar'; end
+
+ protected
+ def output_directory
+ File.join(TmpFile.tmproot, "mongodump")
+ end
+ end
+ end
+end
View
22 lib/astrails/safe/s3.rb
@@ -17,9 +17,6 @@ def save
# FIXME: user friendly error here :)
raise RuntimeError, "pipe-streaming not supported for S3." unless @backup.path
- # needed in cleanup even on dry run
- AWS::S3::Base.establish_connection!(:access_key_id => key, :secret_access_key => secret, :use_ssl => true) unless $LOCAL
-
puts "Uploading #{bucket}:#{full_path}" if $_VERBOSE || $DRY_RUN
unless $DRY_RUN || $LOCAL
if File.stat(@backup.path).size > MAX_S3_FILE_SIZE
@@ -27,9 +24,8 @@ def save
return
end
benchmark = Benchmark.realtime do
- AWS::S3::Bucket.create(bucket)
File.open(@backup.path) do |file|
- AWS::S3::S3Object.store(full_path, file, bucket)
+ remote_bucket.write(file)
end
end
puts "...done" if $_VERBOSE
@@ -43,17 +39,23 @@ def cleanup
return unless keep = @config[:keep, :s3]
puts "listing files: #{bucket}:#{base}*" if $_VERBOSE
- files = AWS::S3::Bucket.objects(bucket, :prefix => base, :max_keys => keep * 2)
+ files = remote_bucket.objects.with_prefix(:prefix => base)
puts files.collect {|x| x.key} if $_VERBOSE
- files = files.
- collect {|x| x.key}.
- sort
+ files = files.sort { |x,y| x.key <=> y.key }
cleanup_with_limit(files, keep) do |f|
puts "removing s3 file #{bucket}:#{f}" if $DRY_RUN || $_VERBOSE
- AWS::S3::Bucket.find(bucket)[f].delete unless $DRY_RUN || $LOCAL
+ f.delete unless $DRY_RUN || $LOCAL
+ end
+ end
+
+ def remote_bucket
+ unless @remote_bucket
+ s3 = AWS::S3.new(:access_key_id => key, :secret_access_key => secret)
+ @remote_bucket = s3.buckets.create(bucket)
end
+ @remote_bucket
end
def bucket
View
15 spec/unit/config_spec.rb
@@ -105,6 +105,12 @@
end
end
+ mongodump do
+ host "host"
+ database "database"
+ user "user"
+ password "password"
+ end
end
expected = {
@@ -183,6 +189,15 @@
"misc" => { "files" => ["/backup/*.rb"] },
},
},
+
+ "mongodump" => {
+ "host" => "host",
+ "databases" => {
+ "database" => {}
+ },
+ "user" => "user",
+ "password" => "password"
+ }
}
config.to_hash.should == expected
View
54 spec/unit/mongodump_spec.rb
@@ -0,0 +1,54 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Astrails::Safe::Mongodump do
+ def def_config
+ {
+ :host => 'prod.example.com',
+ :user => 'testuser',
+ :password => 'p4ssw0rd',
+ }
+ end
+
+ def mongodump(id = :foo, config = def_config)
+ Astrails::Safe::Mongodump.new(id, Astrails::Safe::Config::Node.new(nil, config))
+ end
+
+ before(:each) do
+ stub(Time).now.stub!.strftime {"NOW"}
+ @output_folder = File.join(Astrails::Safe::TmpFile.tmproot, 'mongodump')
+ end
+
+ after(:each) { Astrails::Safe::TmpFile.cleanup }
+
+ describe :backup do
+ before(:each) do
+ @mongo = mongodump
+ end
+
+ {
+ :id => "foo",
+ :kind => "mongodump",
+ :extension => ".tar",
+ :filename => "mongodump-foo.NOW"
+ }.each do |k, v|
+ it "should set #{k} to #{v}" do
+ @mongo.backup.send(k).should == v
+ end
+ end
+
+ it "should set the command" do
+ @mongo.backup.send(:command).should == "mongodump -q \"{xxxx : { \\$ne : 0 } }\" --db foo --host prod.example.com -u testuser -p p4ssw0rd --out #{@output_folder} && cd #{@output_folder} && tar cf - ."
+ end
+
+ {
+ :host => "--host ",
+ :user => "-u ",
+ :password => "-p "
+ }.each do |key, v|
+ it "should not add #{key} to command if it is not present" do
+ @mongo = mongodump(:foo, def_config.reject! {|k,v| k == key})
+ @mongo.backup.send(:command).should_not =~ /#{v}/
+ end
+ end
+ end
+end
View
33 spec/unit/s3_spec.rb
@@ -5,7 +5,7 @@
def def_config
{
:s3 => {
- :bucket => "_bucket",
+ :bucket => "bucket_name",
:key => "_key",
:secret => "_secret",
},
@@ -37,10 +37,12 @@ def s3(config = def_config, backup = def_backup)
before(:each) do
@s3 = s3
- @files = [4,1,3,2].map { |i| stub(o = {}).key {"aaaaa#{i}"}; o }
+ @files = [4,1,3,2].map do |i|
+ stub(object = Object.new).key { "aaaaa#{i}" }
+ object
+ end
- stub(AWS::S3::Bucket).objects("_bucket", :prefix => "_kind/_id/_kind-_id.", :max_keys => 4) {@files}
- stub(AWS::S3::Bucket).find("_bucket").stub![anything].stub!.delete
+ stub(@s3).remote_bucket.stub!.objects.stub!.with_prefix(:prefix => '_kind/_id/_kind-_id.') { @files }
end
it "should check [:keep, :s3]" do
@@ -50,8 +52,10 @@ def s3(config = def_config, backup = def_backup)
end
it "should delete extra files" do
- mock(AWS::S3::Bucket).find("_bucket").mock!["aaaaa1"].mock!.delete
- mock(AWS::S3::Bucket).find("_bucket").mock!["aaaaa2"].mock!.delete
+ dont_allow(@files[0]).delete
+ mock(@files[1]).delete
+ dont_allow(@files[2]).delete
+ mock(@files[3]).delete
@s3.send :cleanup
end
@@ -107,16 +111,14 @@ def s3(config = def_config, backup = def_backup)
def add_stubs(*stubs)
stubs.each do |s|
case s
- when :connection
- stub(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true)
when :stat
stub(File).stat("foo").stub!.size {123}
when :create_bucket
- stub(AWS::S3::Bucket).create
+ stub.instance_of(AWS::S3).buckets.stub!.create.stub!
when :file_open
stub(File).open("foo") {|f, block| block.call(:opened_file)}
when :s3_store
- stub(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket")
+ stub.instance_of(AWS::S3).buckets.stub!.create.stub!.write(:opened_file)
end
end
end
@@ -132,25 +134,26 @@ def add_stubs(*stubs)
end
it "should establish s3 connection" do
- mock(AWS::S3::Base).establish_connection!(:access_key_id => "_key", :secret_access_key => "_secret", :use_ssl => true)
+ connection = AWS::S3.new(:access_key_id => "_key", :secret_access_key => "_secret")
+ mock(AWS::S3).new(:access_key_id => "_key", :secret_access_key => "_secret") { connection }
add_stubs(:stat, :create_bucket, :file_open, :s3_store)
+
@s3.send(:save)
end
it "should open local file" do
- add_stubs(:connection, :stat, :create_bucket)
+ add_stubs(:stat, :create_bucket)
mock(File).open("foo")
@s3.send(:save)
end
it "should upload file" do
- add_stubs(:connection, :stat, :create_bucket, :file_open)
- mock(AWS::S3::S3Object).store(@full_path, :opened_file, "_bucket")
+ add_stubs(:stat, :create_bucket, :file_open)
+ mock(@s3).remote_bucket.mock!.write(:opened_file)
@s3.send(:save)
end
it "should fail on files bigger then 5G" do
- add_stubs(:connection)
mock(File).stat("foo").stub!.size {5*1024*1024*1024+1}
mock(STDERR).puts(anything)
dont_allow(Benchmark).realtime

No commit comments for this range

Something went wrong with that request. Please try again.