Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Make is possible to backup the MongoDB data directory directly without using mongodump #403

Closed
wants to merge 2 commits into from

1 participant

@benmccann

It is much preferred to backup the mongodb data directory directly as compared to using mongodump especially as database size grows because it requires less free disk space be available and is much faster.

@benmccann

Going to hold off on this for now until the acceptance tests are ready

@benmccann benmccann closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Feb 3, 2013
  1. @benmccann
Commits on Feb 9, 2013
  1. @benmccann

    Fix mongodb backup

    benmccann authored
This page is out of date. Refresh to see the latest.
View
4 README.md
@@ -432,6 +432,10 @@ View the [issue log](https://github.com/meskyanichi/backup/issues) and post them
<td><a href="https://github.com/SteveNewson" target="_blank">Steve Newson ( SteveNewson )</a></td>
<td>Pushover Notifier</td>
</tr>
+ <tr>
+ <td><a href="https://github.com/benmccann" target="_blank">Ben McCann ( benmccann )</a></td>
+ <td>MongoDB data directory backup</td>
+ </tr>
</table>
View
16 lib/backup/database/base.rb
@@ -11,6 +11,14 @@ class Base
# * Called using super(model) from subclasses *
def initialize(model)
@model = model
+
+ @dump_path = File.join(
+ Config.tmp_path,
+ @model.trigger,
+ 'databases',
+ self.class.name.split('::').last
+ )
+
load_defaults!
end
@@ -25,14 +33,8 @@ def perform!
private
##
- # Defines the @dump_path and ensures it exists by creating it
+ # Ensures the @dump_path exists by creating it
def prepare!
- @dump_path = File.join(
- Config.tmp_path,
- @model.trigger,
- 'databases',
- self.class.name.split('::').last
- )
FileUtils.mkdir_p(@dump_path)
end
View
57 lib/backup/database/mongodb.rb
@@ -21,12 +21,24 @@ class MongoDB < Base
attr_accessor :ipv6
##
+ # Use mongodump
+ attr_accessor :use_mongodump
+
+ ##
+ # Path where DB files are stored
+ attr_accessor :db_path
+
+ ##
+ # Path where the backup is stored
+ attr_accessor :output_path
+
+ ##
# Collections to dump, collections that aren't specified won't get dumped
attr_accessor :only_collections
##
# Additional "mongodump" options
- attr_accessor :additional_options
+ attr_accessor :mongodump_options
##
# Path to the mongodump utility (optional)
@@ -36,6 +48,10 @@ class MongoDB < Base
:message => 'Use MongoDB#mongodump_utility instead.',
:action => lambda {|klass, val| klass.mongodump_utility = val }
+ attr_deprecate :additional_options, :version => '3.0.28',
+ :message => 'Use MongoDB#mongodump_options instead.',
+ :action => lambda {|klass, val| klass.mongodump_options = val }
+
##
# Path to the mongo utility (optional)
attr_accessor :mongo_utility
@@ -49,9 +65,16 @@ class MongoDB < Base
def initialize(model, &block)
super(model)
+ timestamp = Time.now.to_i.to_s[-5, 5]
+
@only_collections ||= Array.new
- @additional_options ||= Array.new
+ @mongodump_options ||= Array.new
@ipv6 ||= false
+ @use_mongodump ||= false
+ @db_path ||= '/var/lib/mongodb'
+ @output_path ||= @use_mongodump ?
+ @dump_path + '-' + timestamp + '.tar' :
+ File.join(File.expand_path(ENV['HOME'] || ''), 'backups','mongodb-' + timestamp + '.tar')
@lock ||= false
instance_eval(&block) if block_given?
@@ -70,7 +93,10 @@ def perform!
super
lock_database if @lock
- @only_collections.empty? ? dump! : specific_collection_dump!
+
+ if @use_mongodump
+ @only_collections.empty? ? dump! : specific_collection_dump!
+ end
rescue => err
raise Errors::Database::MongoDBError.wrap(err, 'Database Dump Failed!')
@@ -115,35 +141,36 @@ def mongodump
def package!
return unless @model.compressor
+ data_path = @use_mongodump ? @dump_path : @db_path
pipeline = Pipeline.new
- base_dir = File.dirname(@dump_path)
- dump_dir = File.basename(@dump_path)
- timestamp = Time.now.to_i.to_s[-5, 5]
- outfile = @dump_path + '-' + timestamp + '.tar'
+ base_dir = File.dirname(data_path)
+ data_dir = File.basename(data_path)
+
+ FileUtils.mkpath(File.dirname(@output_path))
Logger.message(
"#{ database_name } started compressing and packaging:\n" +
- " '#{ @dump_path }'"
+ " '#{ data_path }'"
)
- pipeline << "#{ utility(:tar) } -cf - -C '#{ base_dir }' '#{ dump_dir }'"
+ pipeline << "#{ utility(:tar) } -cf - -C '#{ base_dir }' '#{ data_dir }'"
@model.compressor.compress_with do |command, ext|
pipeline << command
- outfile << ext
+ @output_path << ext
end
- pipeline << "cat > #{ outfile }"
+ pipeline << "cat > #{ @output_path }"
pipeline.run
if pipeline.success?
Logger.message(
"#{ database_name } completed compressing and packaging:\n" +
- " '#{ outfile }'"
+ " '#{ @output_path }'"
)
FileUtils.rm_rf(@dump_path)
else
raise Errors::Database::PipelineError,
- "#{ database_name } Failed to create compressed dump package:\n" +
- "'#{ outfile }'\n" +
+ "#{ database_name } Failed to create compressed package:\n" +
+ "'#{ @output_path }'\n" +
pipeline.error_messages
end
end
@@ -184,7 +211,7 @@ def ipv6_option
# Builds a MongoDB compatible string for the
# additional options specified by the user
def user_options
- @additional_options.join(' ')
+ @mongodump_options.join(' ')
end
##
View
29 spec/database/mongodb_spec.rb
@@ -13,8 +13,10 @@
db.port = 123
db.ipv6 = true
+ db.use_mongodump = true
+ db.db_path = '/var/lib/mongodb'
db.only_collections = ['users', 'pirates']
- db.additional_options = ['--query', '--foo']
+ db.mongodump_options = ['--query', '--foo']
db.mongodump_utility = '/path/to/mongodump'
db.mongo_utility = '/path/to/mongo'
db.lock = true
@@ -47,8 +49,10 @@
db.port.should == 123
db.ipv6.should == true
+ db.use_mongodump.should == true
+ db.db_path.should == '/var/lib/mongodb'
db.only_collections.should == ['users', 'pirates']
- db.additional_options.should == ['--query', '--foo']
+ db.mongodump_options.should == ['--query', '--foo']
db.mongodump_utility.should == '/path/to/mongodump'
db.mongo_utility.should == '/path/to/mongo'
db.lock.should == true
@@ -73,8 +77,10 @@
db.port.should be_nil
db.ipv6.should be_false
+ db.use_mongodump.should be_false
db.only_collections.should == []
- db.additional_options.should == []
+ db.mongodump_options.should == []
+ db.db_path.should == '/var/lib/mongodb'
db.mongodump_utility.should == '/real/mongodump'
db.mongo_utility.should == '/real/mongo'
db.lock.should be_false
@@ -92,8 +98,9 @@
db.port = 789
db.ipv6 = 'default_ipv6'
+ db.use_mongodump = true
db.only_collections = ['collection']
- db.additional_options = ['--opt']
+ db.mongodump_options = ['--opt']
db.mongodump_utility = '/default/path/to/mongodump'
db.mongo_utility = '/default/path/to/mongo'
db.lock = 'default_lock'
@@ -111,8 +118,9 @@
db.port.should == 123
db.ipv6.should == true
+ db.use_mongodump.should == true
db.only_collections.should == ['users', 'pirates']
- db.additional_options.should == ['--query', '--foo']
+ db.mongodump_options.should == ['--query', '--foo']
db.mongodump_utility.should == '/path/to/mongodump'
db.mongo_utility.should == '/path/to/mongo'
db.lock.should == true
@@ -130,8 +138,9 @@
db.port.should == 789
db.ipv6.should == 'default_ipv6'
+ db.use_mongodump.should == true
db.only_collections.should == ['collection']
- db.additional_options.should == ['--opt']
+ db.mongodump_options.should == ['--opt']
db.mongodump_utility.should == '/default/path/to/mongodump'
db.mongo_utility.should == '/default/path/to/mongo'
db.lock.should == 'default_lock'
@@ -341,7 +350,7 @@
end.to raise_error(
Backup::Errors::Database::PipelineError,
"Database::PipelineError: Database::MongoDB " +
- "Failed to create compressed dump package:\n" +
+ "Failed to create compressed package:\n" +
" '/path/to/dump/folder-#{ timestamp }.tar.gz'\n" +
" pipeline_errors"
)
@@ -405,15 +414,15 @@
end
describe '#user_options' do
- context 'when #additional_options are set' do
+ context 'when #mongodump_options are set' do
it 'should return the command string for the options' do
db.send(:user_options).should == '--query --foo'
end
end
- context 'when #additional_options are not set' do
+ context 'when #mongodump_options are not set' do
it 'should return an empty string' do
- db.additional_options = []
+ db.mongodump_options = []
db.send(:user_options).should == ''
end
end
View
3  templates/cli/utility/database/mongodb
@@ -8,8 +8,9 @@
db.host = "localhost"
db.port = 5432
db.ipv6 = false
+ db.use_mongodump = false
db.only_collections = ["only", "these" "collections"]
- db.additional_options = []
+ db.mongodump_options = []
db.lock = false
# Optional: Use to set the location of these utilities
# if they cannot be found by their name in your $PATH
Something went wrong with that request. Please try again.