Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merge branch 'develop'

  • Loading branch information...
commit 54a0acf744d20d76fad42f8e976a8e4882d2f316 2 parents 479f3f6 + cc8dc86
@meskyanichi meskyanichi authored
Showing with 10,505 additions and 5,943 deletions.
  1. +1 −5 Gemfile
  2. +46 −50 Gemfile.lock
  3. +54 −27 README.md
  4. +16 −39 lib/backup.rb
  5. +42 −18 lib/backup/archive.rb
  6. +110 −25 lib/backup/cleaner.rb
  7. +17 −32 lib/backup/cli/helpers.rb
  8. +46 −107 lib/backup/cli/utility.rb
  9. +14 −2 lib/backup/compressor/base.rb
  10. +10 −24 lib/backup/compressor/bzip2.rb
  11. +10 −24 lib/backup/compressor/gzip.rb
  12. +10 −23 lib/backup/compressor/lzma.rb
  13. +12 −32 lib/backup/compressor/pbzip2.rb
  14. +171 −0 lib/backup/config.rb
  15. +1 −2  lib/backup/configuration/compressor/base.rb
  16. +4 −4 lib/backup/configuration/compressor/pbzip2.rb
  17. +2 −1  lib/backup/configuration/database/base.rb
  18. +8 −0 lib/backup/configuration/database/mongodb.rb
  19. +4 −0 lib/backup/configuration/database/mysql.rb
  20. +4 −0 lib/backup/configuration/database/postgresql.rb
  21. +4 −0 lib/backup/configuration/database/redis.rb
  22. +5 −1 lib/backup/configuration/database/riak.rb
  23. +1 −2  lib/backup/configuration/encryptor/base.rb
  24. +1 −1  lib/backup/configuration/encryptor/open_ssl.rb
  25. +7 −2 lib/backup/configuration/helpers.rb
  26. +4 −28 lib/backup/configuration/notifier/base.rb
  27. +1 −1  lib/backup/configuration/storage/base.rb
  28. +14 −4 lib/backup/configuration/storage/dropbox.rb
  29. +10 −0 lib/backup/configuration/syncer/base.rb
  30. +0 −45 lib/backup/configuration/syncer/rsync.rb
  31. +28 −0 lib/backup/configuration/syncer/rsync/base.rb
  32. +11 −0 lib/backup/configuration/syncer/rsync/local.rb
  33. +11 −0 lib/backup/configuration/syncer/rsync/pull.rb
  34. +31 −0 lib/backup/configuration/syncer/rsync/push.rb
  35. +0 −4 lib/backup/configuration/syncer/s3.rb
  36. +25 −7 lib/backup/database/base.rb
  37. +112 −75 lib/backup/database/mongodb.rb
  38. +54 −29 lib/backup/database/mysql.rb
  39. +60 −42 lib/backup/database/postgresql.rb
  40. +61 −39 lib/backup/database/redis.rb
  41. +35 −11 lib/backup/database/riak.rb
  42. +4 −5 lib/backup/dependency.rb
  43. +13 −1 lib/backup/encryptor/base.rb
  44. +39 −39 lib/backup/encryptor/gpg.rb
  45. +28 −38 lib/backup/encryptor/open_ssl.rb
  46. +0 −87 lib/backup/finder.rb
  47. +20 −11 lib/backup/logger.rb
  48. +206 −163 lib/backup/model.rb
  49. +27 −25 lib/backup/notifier/base.rb
  50. +7 −13 lib/backup/notifier/campfire.rb
  51. +28 −28 lib/backup/notifier/hipchat.rb
  52. +24 −26 lib/backup/notifier/mail.rb
  53. +10 −18 lib/backup/notifier/presently.rb
  54. +9 −17 lib/backup/notifier/prowl.rb
  55. +11 −18 lib/backup/notifier/twitter.rb
  56. +47 −0 lib/backup/package.rb
  57. +81 −16 lib/backup/packager.rb
  58. +48 −35 lib/backup/splitter.rb
  59. +44 −172 lib/backup/storage/base.rb
  60. +31 −46 lib/backup/storage/cloudfiles.rb
  61. +117 −0 lib/backup/storage/cycler.rb
  62. +92 −76 lib/backup/storage/dropbox.rb
  63. +30 −40 lib/backup/storage/ftp.rb
  64. +44 −45 lib/backup/storage/local.rb
  65. +55 −49 lib/backup/storage/ninefold.rb
  66. +0 −47 lib/backup/storage/object.rb
  67. +49 −56 lib/backup/storage/rsync.rb
  68. +33 −44 lib/backup/storage/s3.rb
  69. +21 −48 lib/backup/storage/scp.rb
  70. +26 −40 lib/backup/storage/sftp.rb
  71. +7 −0 lib/backup/syncer/base.rb
  72. +0 −152 lib/backup/syncer/rsync.rb
  73. +78 −0 lib/backup/syncer/rsync/base.rb
  74. +53 −0 lib/backup/syncer/rsync/local.rb
  75. +38 −0 lib/backup/syncer/rsync/pull.rb
  76. +113 −0 lib/backup/syncer/rsync/push.rb
  77. +42 −32 lib/backup/syncer/s3.rb
  78. +1 −1  lib/backup/version.rb
  79. +235 −69 spec/archive_spec.rb
  80. +0 −11 spec/backup_spec.rb
  81. +304 −0 spec/cleaner_spec.rb
  82. +142 −1 spec/cli/helpers_spec.rb
  83. +338 −13 spec/cli/utility_spec.rb
  84. +31 −0 spec/compressor/base_spec.rb
  85. +60 −35 spec/compressor/bzip2_spec.rb
  86. +60 −35 spec/compressor/gzip_spec.rb
  87. +60 −35 spec/compressor/lzma_spec.rb
  88. +98 −37 spec/compressor/pbzip2_spec.rb
  89. +321 −0 spec/config_spec.rb
  90. +4 −4 spec/configuration/base_spec.rb
  91. +1 −0  spec/configuration/compressor/bzip2_spec.rb
  92. +1 −0  spec/configuration/compressor/gzip_spec.rb
  93. +1 −0  spec/configuration/compressor/lzma_spec.rb
  94. +32 −0 spec/configuration/compressor/pbzip2_spec.rb
  95. +2 −1  spec/configuration/database/base_spec.rb
  96. +26 −16 spec/configuration/database/mongodb_spec.rb
  97. +4 −0 spec/configuration/database/mysql_spec.rb
  98. +4 −0 spec/configuration/database/postgresql_spec.rb
  99. +4 −0 spec/configuration/database/redis_spec.rb
  100. +4 −0 spec/configuration/database/riak_spec.rb
  101. +1 −0  spec/configuration/encryptor/gpg_spec.rb
  102. +1 −0  spec/configuration/encryptor/open_ssl_spec.rb
  103. +32 −0 spec/configuration/notifier/base_spec.rb
  104. +1 −0  spec/configuration/notifier/campfire_spec.rb
  105. +1 −0  spec/configuration/notifier/hipchat_spec.rb
  106. +1 −0  spec/configuration/notifier/mail_spec.rb
  107. +1 −0  spec/configuration/notifier/presently_spec.rb
  108. +1 −0  spec/configuration/notifier/prowl_spec.rb
  109. +1 −0  spec/configuration/notifier/twitter_spec.rb
  110. +1 −0  spec/configuration/storage/cloudfiles_spec.rb
  111. +4 −3 spec/configuration/storage/dropbox_spec.rb
  112. +1 −0  spec/configuration/storage/ftp_spec.rb
  113. +1 −0  spec/configuration/storage/local_spec.rb
  114. +1 −0  spec/configuration/storage/ninefold_spec.rb
  115. +3 −1 spec/configuration/storage/rsync_spec.rb
  116. +1 −0  spec/configuration/storage/s3_spec.rb
  117. +1 −0  spec/configuration/storage/scp_spec.rb
  118. +1 −0  spec/configuration/storage/sftp_spec.rb
  119. +33 −0 spec/configuration/syncer/rsync/base_spec.rb
  120. +10 −0 spec/configuration/syncer/rsync/local_spec.rb
  121. +10 −0 spec/configuration/syncer/rsync/pull_spec.rb
  122. +12 −15 spec/configuration/syncer/{rsync_spec.rb → rsync/push_spec.rb}
  123. +2 −3 spec/configuration/syncer/s3_spec.rb
  124. +35 −20 spec/database/base_spec.rb
  125. +298 −119 spec/database/mongodb_spec.rb
  126. +147 −72 spec/database/mysql_spec.rb
  127. +155 −100 spec/database/postgresql_spec.rb
  128. +200 −97 spec/database/redis_spec.rb
  129. +82 −24 spec/database/riak_spec.rb
  130. +49 −0 spec/dependency_spec.rb
  131. +30 −0 spec/encryptor/base_spec.rb
  132. +105 −28 spec/encryptor/gpg_spec.rb
  133. +85 −114 spec/encryptor/open_ssl_spec.rb
  134. +0 −91 spec/finder_spec.rb
  135. +74 −8 spec/logger_spec.rb
  136. +528 −220 spec/model_spec.rb
  137. +89 −0 spec/notifier/base_spec.rb
  138. +147 −119 spec/notifier/campfire_spec.rb
  139. +140 −145 spec/notifier/hipchat_spec.rb
  140. +190 −248 spec/notifier/mail_spec.rb
  141. +147 −282 spec/notifier/presently_spec.rb
  142. +79 −111 spec/notifier/prowl_spec.rb
  143. +87 −106 spec/notifier/twitter_spec.rb
  144. +61 −0 spec/package_spec.rb
  145. +154 −0 spec/packager_spec.rb
  146. +36 −13 spec/spec_helper.rb
  147. +90 −41 spec/splitter_spec.rb
  148. +95 −239 spec/storage/base_spec.rb
  149. +185 −75 spec/storage/cloudfiles_spec.rb
  150. +239 −0 spec/storage/cycler_spec.rb
  151. +318 −87 spec/storage/dropbox_spec.rb
  152. +165 −152 spec/storage/ftp_spec.rb
  153. +206 −54 spec/storage/local_spec.rb
  154. +264 −128 spec/storage/ninefold_spec.rb
  155. +0 −74 spec/storage/object_spec.rb
  156. +244 −163 spec/storage/rsync_spec.rb
  157. +175 −64 spec/storage/s3_spec.rb
  158. +156 −150 spec/storage/scp_spec.rb
  159. +153 −135 spec/storage/sftp_spec.rb
  160. +22 −0 spec/syncer/base_spec.rb
  161. +118 −0 spec/syncer/rsync/base_spec.rb
  162. +121 −0 spec/syncer/rsync/local_spec.rb
  163. +90 −0 spec/syncer/rsync/pull_spec.rb
  164. +327 −0 spec/syncer/rsync/push_spec.rb
  165. +0 −195 spec/syncer/rsync_spec.rb
  166. +180 −91 spec/syncer/s3_spec.rb
  167. +1 −1  templates/cli/utility/config
  168. +4 −0 templates/cli/utility/database/mongodb
  169. +3 −0  templates/cli/utility/database/mysql
  170. +3 −0  templates/cli/utility/database/postgresql
  171. +3 −0  templates/cli/utility/database/redis
  172. +3 −0  templates/cli/utility/database/riak
  173. +4 −1 templates/cli/utility/storage/dropbox
  174. +12 −0 templates/cli/utility/syncer/rsync_local
  175. +2 −2 templates/cli/utility/syncer/{rsync → rsync_pull}
  176. +17 −0 templates/cli/utility/syncer/rsync_push
  177. +1 −1  templates/storage/dropbox/authorization_url.erb
View
6 Gemfile
@@ -6,12 +6,8 @@ source 'http://rubygems.org'
# Include gem dependencies from the gemspec for development purposes
gemspec
-# Load Backup::Dependency
-["cli/helpers", "dependency"].each do |library|
- require File.expand_path("../lib/backup/#{library}", __FILE__)
-end
-
# Dynamically define the dependencies specified in Backup::Dependency.all
+require File.expand_path("../lib/backup/dependency", __FILE__)
Backup::Dependency.all.each do |name, gemspec|
gem(name, gemspec[:version])
end
View
96 Gemfile.lock
@@ -1,7 +1,7 @@
PATH
remote: .
specs:
- backup (3.0.19)
+ backup (3.0.21)
POpen4 (~> 0.1.4)
thor (~> 0.14.6)
@@ -12,82 +12,80 @@ GEM
Platform (>= 0.4.0)
open4
Platform (0.4.0)
+ activesupport (3.1.3)
+ multi_json (~> 1.0)
addressable (2.2.6)
builder (3.0.0)
crack (0.1.8)
- diff-lcs (1.1.2)
- dropbox (1.3.0)
- json (>= 1.2.0)
- multipart-post (>= 1.1.0)
- oauth (>= 0.3.6)
- excon (0.6.6)
- faraday (0.7.4)
+ diff-lcs (1.1.3)
+ dropbox-sdk (1.1)
+ json
+ excon (0.9.4)
+ faraday (0.7.5)
addressable (~> 2.2.6)
- multipart-post (~> 1.1.0)
+ multipart-post (~> 1.1.3)
rack (>= 1.1.0, < 2)
- faraday_middleware (0.7.0)
- faraday (~> 0.7.3)
- ffi (1.0.9)
- fog (0.11.0)
+ ffi (1.0.11)
+ fog (1.1.2)
builder
- excon (~> 0.6.5)
+ excon (~> 0.9.0)
formatador (~> 0.2.0)
mime-types
multi_json (~> 1.0.3)
net-scp (~> 1.0.4)
- net-ssh (~> 2.1.4)
+ net-ssh (>= 2.1.3)
nokogiri (~> 1.5.0)
ruby-hmac
formatador (0.2.1)
- fuubar (0.0.5)
+ fuubar (0.0.6)
rspec (~> 2.0)
- rspec-instafail (~> 0.1.4)
+ rspec-instafail (~> 0.1.8)
ruby-progressbar (~> 0.0.10)
growl (1.0.3)
- guard (0.3.4)
+ guard (0.10.0)
+ ffi (>= 0.5.0)
thor (~> 0.14.6)
- guard-rspec (0.3.1)
- guard (>= 0.2.2)
- hashie (1.1.0)
+ guard-rspec (0.6.0)
+ guard (>= 0.10.0)
hipchat (0.4.1)
httparty
- httparty (0.7.4)
+ httparty (0.7.8)
crack (= 0.1.8)
i18n (0.6.0)
json (1.5.4)
- libnotify (0.5.5)
+ libnotify (0.7.1)
mail (2.3.0)
i18n (>= 0.4.0)
mime-types (~> 1.16)
treetop (~> 1.4.8)
- mime-types (1.16)
- mocha (0.9.12)
- multi_json (1.0.3)
- multi_xml (0.3.0)
- multipart-post (1.1.3)
+ metaclass (0.0.1)
+ mime-types (1.17.2)
+ mocha (0.10.0)
+ metaclass (~> 0.0.1)
+ multi_json (1.0.4)
+ multipart-post (1.1.4)
net-scp (1.0.4)
net-ssh (>= 1.99.1)
net-sftp (2.0.5)
net-ssh (>= 2.0.9)
net-ssh (2.1.4)
nokogiri (1.5.0)
- oauth (0.4.5)
- open4 (1.2.0)
- polyglot (0.3.2)
+ open4 (1.3.0)
+ polyglot (0.3.3)
prowler (1.3.1)
- rack (1.3.2)
- rb-fsevent (0.4.0)
- rb-inotify (0.8.5)
+ rack (1.4.0)
+ rb-fsevent (0.4.3.1)
+ rb-inotify (0.8.8)
ffi (>= 0.5.0)
- rspec (2.5.0)
- rspec-core (~> 2.5.0)
- rspec-expectations (~> 2.5.0)
- rspec-mocks (~> 2.5.0)
- rspec-core (2.5.1)
- rspec-expectations (2.5.0)
+ rspec (2.8.0)
+ rspec-core (~> 2.8.0)
+ rspec-expectations (~> 2.8.0)
+ rspec-mocks (~> 2.8.0)
+ rspec-core (2.8.0)
+ rspec-expectations (2.8.0)
diff-lcs (~> 1.1.2)
- rspec-instafail (0.1.7)
- rspec-mocks (2.5.0)
+ rspec-instafail (0.1.9)
+ rspec-mocks (2.8.0)
ruby-hmac (0.4.0)
ruby-progressbar (0.0.10)
simple_oauth (0.1.5)
@@ -96,20 +94,18 @@ GEM
treetop (1.4.10)
polyglot
polyglot (>= 0.3.1)
- twitter (1.7.1)
- faraday (~> 0.7.4)
- faraday_middleware (~> 0.7.0)
- hashie (~> 1.1.0)
- multi_json (~> 1.0.0)
- multi_xml (~> 0.3.0)
- simple_oauth (~> 0.1.5)
+ twitter (2.0.2)
+ activesupport (>= 2.3.9, < 4)
+ faraday (~> 0.7)
+ multi_json (~> 1.0)
+ simple_oauth (~> 0.1)
PLATFORMS
ruby
DEPENDENCIES
backup!
- dropbox (~> 1.3.0)
+ dropbox-sdk (~> 1.1.0)
fog (>= 0.11.0)
fuubar
growl
View
81 README.md
@@ -15,12 +15,6 @@ Author
Drop me a message for any questions, suggestions, requests, bugs or submit them to the [issue log](https://github.com/meskyanichi/backup/issues).
-Core Contributor
-----------------
-
-**[Brian Burns](https://github.com/burns)**
-
-
Installation
------------
@@ -92,14 +86,14 @@ Storage Features
- **Incremental Backups, applies to:**
- Remote Servers *(Only Protocols: RSync)*
-[Storage Wiki Page](https://github.com/meskyanichi/backup/wiki/Storages)
+[Cycling Wiki Page](https://github.com/meskyanichi/backup/wiki/Cycling)
[Splitter Wiki Page](https://github.com/meskyanichi/backup/wiki/Splitter)
Syncers
-------
-- RSync
+- RSync (Push, Pull and Local)
- Amazon Simple Storage Service (S3)
[Syncer Wiki Page](https://github.com/meskyanichi/backup/wiki/Syncers)
@@ -240,21 +234,50 @@ end
### Brief explanation for the above example configuration
-It will dump two databases (MySQL and MongoDB), it'll create two (.t)archives (user_avatars and logs). It'll package the two database and two archives together in a single (.t)archive. It'll run the Gzip compressor to compress that archive, and then it'll run the OpenSSL encryptor to encrypt the compressed archive. Then that encrypted archive will be stored to your Amazon S3 account. If all goes well, and no exceptions are raised, you'll be notified via the Twitter notifier that the backup succeeded. If there was an exception raised during the backup process, then you'd receive an email in your inbox containing detailed exception information, as well as receive a simple Twitter message that something went wrong.
-
-Aside of S3, we have also defined two `SFTP` storage methods, and given them two unique identifiers `Server A` and `Server B` to distinguish between the two. With these in place, a copy of the backup will now also be stored on two separate servers: `a.my-backup-server.com` and `b.my-backup-server.com`.
-
-As you can see, you can freely mix and match **archives**, **databases**, **compressors**, **encryptors**, **storages** and **notifiers** for your backups. You could even specify 4 storage locations if you wanted: Amazon S3, Rackspace Cloud Files, Ninefold and Dropbox, it'd then store your packaged backup to 4 separate locations for high redundancy. This also applies to compressors (like Gzip, Bzip2, Lzma) and encryptors, you could double encrypt your backup with OpenSSL followed by GPG if you wanted.
-
-Also, notice the `split_into_chunks_of 4000` at the top of the configuration. This tells Backup to split any backups that exceed in 4000 MEGABYTES of size in to multiple smaller chunks. Assuming your backup file is 12000 MEGABYTES (12GB) in size, then Backup will go ahead and split it in to 3 chunks of 4000 MEGABYTES and transfer them individually. This is useful for when you are using Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with this, the backup file size is no longer a constraint.
-
-Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If you simply wanted to sync to a separate backup server that you own, you could also use the RSync syncer for even more efficient backups that only transfer the **bytes** of each file that changed.
-
-There are more **archives**, **databases**, **compressors**, **encryptors**, **storages** and **notifiers** than displayed in the example, all available components are listed at the top of this README, as well as in the [Wiki](https://github.com/meskyanichi/backup/wiki) with more detailed information.
+First, it will dump the two Databases (MySQL and MongoDB). The MySQL dump will be piped through the Gzip Compressor into
+`sample_backup/databases/MySQL/my_sample_mysql_db.sql.gz`. The MongoDB dump will be dumped into
+`sample_backup/databases/MongoDB/`, which will then be packaged into `sample_backup/databases/MongoDB-#####.tar.gz`
+(`#####` will be a simple unique identifier, in case multiple dumps are performed.)
+Next, it will create two _tar_ Archives (user_avatars and logs). Each will be piped through the Gzip Compressor into
+`sample_backup/archives/` as `user_archives.tar.gz` and `logs.tar.gz`.
+Finally, the `sample_backup` directory will be packaged into an uncompressed _tar_ archive, which will be piped through
+the OpenSSL Encryptor to encrypt this final package into `YYYY-MM-DD-hh-mm-ss.sample_backup.tar.enc`. This final
+encrypted archive will then be transfered to your Amazon S3 account. If all goes well, and no exceptions are raised,
+you'll be notified via the Twitter notifier that the backup succeeded. If any warnings were issued or there was an
+exception raised during the backup process, then you'd receive an email in your inbox containing detailed exception
+information, as well as receive a simple Twitter message that something went wrong.
+
+Aside of S3, we have also defined two `SFTP` storage methods, and given them two unique identifiers `Server A` and
+`Server B` to distinguish between the two. With these in place, a copy of the backup will now also be stored on two
+separate servers: `a.my-backup-server.com` and `b.my-backup-server.com`.
+
+As you can see, you can freely mix and match **archives**, **databases**, **compressors**, **encryptors**, **storages**
+and **notifiers** for your backups. You could even specify 4 storage locations if you wanted: Amazon S3, Rackspace Cloud
+Files, Ninefold and Dropbox, it'd then store your packaged backup to 4 separate locations for high redundancy.
+
+Also, notice the `split_into_chunks_of 4000` at the top of the configuration. This tells Backup to split any backups
+that exceed in 4000 MEGABYTES of size in to multiple smaller chunks. Assuming your backup file is 12000 MEGABYTES (12GB)
+in size, then Backup will take the output which was piped from _tar_ into the OpenSSL Compressor and additionally pipe
+that output through the _split_ utility, which will result in 3 chunks of 4000 MEGABYTES with additional file extensions
+of `-aa`, `-ab` and `-ac`. These files will then be individually transfered. This is useful for when you are using
+Amazon S3, Rackspace Cloud Files, or other 3rd party storage services which limit you to "5GB per file" uploads. So with
+this, the backup file size is no longer a constraint.
+
+Additionally we have also defined a **S3 Syncer** ( `sync_with S3` ), which does not follow the above process of
+archiving/compression/encryption, but instead will directly sync the whole `videos` and `music` folder structures from
+your machine to your Amazon S3 account. (very efficient and cost-effective since it will only transfer files that were
+added/changed. Additionally, since we flagged it to 'mirror', it'll also remove files from S3 that no longer exist). If
+you simply wanted to sync to a separate backup server that you own, you could also use the RSync syncer for even more
+efficient backups that only transfer the **bytes** of each file that changed.
+
+There are more **archives**, **databases**, **compressors**, **encryptors**, **storages** and **notifiers** than
+displayed in the example, all available components are listed at the top of this README, as well as in the
+[Wiki](https://github.com/meskyanichi/backup/wiki) with more detailed information.
### Running the example
-Notice the `Backup::Model.new(:sample_backup, 'A sample backup configuration') do` at the top of the above example. The `:sample_backup` is called the **trigger**. This is used to identify the backup procedure/file and initialize it.
+Notice the `Backup::Model.new(:sample_backup, 'A sample backup configuration') do` at the top of the above example. The
+`:sample_backup` is called the **trigger**. This is used to identify the backup procedure/file and initialize it.
``` sh
backup perform -t [--trigger] sample_backup
@@ -264,7 +287,9 @@ Now it'll run the backup, it's as simple as that.
### Automatic backups
-Since Backup is an easy-to-use command line utility, you should write a crontask to invoke it periodically. I recommend using [Whenever](https://github.com/javan/whenever) to manage your crontab. It'll allow you to write to the crontab using pure Ruby, and it provides an elegant DSL to do so. Here's an example:
+Since Backup is an easy-to-use command line utility, you should write a crontask to invoke it periodically. I recommend
+using [Whenever](https://github.com/javan/whenever) to manage your crontab. It'll allow you to write to the crontab
+using pure Ruby, and it provides an elegant DSL to do so. Here's an example:
``` rb
every 6.hours do
@@ -272,12 +297,14 @@ every 6.hours do
end
```
-With this in place, run `whenever --update-crontab backup` to write the equivalent of the above Ruby syntax to the crontab in cron-syntax. Cron will now invoke `backup perform --trigger sample_backup` every 6 hours. Check out the Whenever project page for more information.
+With this in place, run `whenever --update-crontab backup` to write the equivalent of the above Ruby syntax to the
+crontab in cron-syntax. Cron will now invoke `backup perform --trigger sample_backup` every 6 hours. Check out the
+Whenever project page for more information.
Documentation
-------------
-See the [Wiki Pages](https://github.com/meskyanichi/backup/wiki). The subjects labeled **without** the "Backup 2)"-prefix are meant for Backup 3 users.
+See the [Wiki Pages](https://github.com/meskyanichi/backup/wiki).
Suggestions, Bugs, Requests, Questions
@@ -294,6 +321,10 @@ Contributors
<th>Contribution</th>
</tr>
<tr>
+ <td><a href="https://github.com/burns" target="_blank"><b>Brian D. Burns ( burns )</b></a></td>
+ <td><b>Core Contributor</b></td>
+ </tr>
+ <tr>
<td><a href="https://github.com/asanghi" target="_blank">Aditya Sanghi ( asanghi )</a></td>
<td>Twitter Notifier, Dropbox Timeout Configuration</td>
</tr>
@@ -409,10 +440,6 @@ Contributors
<td><a href="https://github.com/szymonpk" target="_blank">Szymon ( szymonpk )</a></td>
<td>Pbzip2 compressor</td>
</tr>
- <tr>
- <td><a href="https://github.com/burns" target="_blank">burns ( burns )</a></td>
- <td>Improved Backup cycling implementation by refreshing all user configuration during the cycle procedure</td>
- </tr>
</table>
View
55 lib/backup.rb
@@ -24,22 +24,6 @@
module Backup
##
- # List the available database, storage, compressor, encryptor and notifier constants.
- # These are used to dynamically define these constants as classes inside Backup::Finder
- # to provide a nicer configuration file DSL syntax to the users. Adding existing constants
- # to the arrays below will enable the user to use a constant instead of a string.
- # Example, instead of:
- # database "MySQL" do |mysql|
- # You can do:
- # database MySQL do |mysql|
- DATABASES = ['MySQL', 'PostgreSQL', 'MongoDB', 'Redis', 'Riak']
- STORAGES = ['S3', 'CloudFiles', 'Ninefold', 'Dropbox', 'FTP', 'SFTP', 'SCP', 'RSync', 'Local']
- COMPRESSORS = ['Gzip', 'Bzip2', 'Pbzip2', 'Lzma']
- ENCRYPTORS = ['OpenSSL', 'GPG']
- SYNCERS = ['RSync', 'S3']
- NOTIFIERS = ['Mail', 'Twitter', 'Campfire', 'Presently', 'Prowl', 'Hipchat']
-
- ##
# Backup's internal paths
LIBRARY_PATH = File.join(File.dirname(__FILE__), 'backup')
CLI_PATH = File.join(LIBRARY_PATH, 'cli')
@@ -53,24 +37,14 @@ module Backup
TEMPLATE_PATH = File.expand_path('../../templates', __FILE__)
##
- # Backup's Environment paths
- USER = ENV['USER'] || Etc.getpwuid.name
- HOME = File.expand_path(ENV['HOME'] || '')
- PATH = File.join(HOME, 'Backup')
- CONFIG_FILE = File.join(PATH, 'config.rb')
- DATA_PATH = File.join(PATH, 'data')
- LOG_PATH = File.join(PATH, 'log')
- CACHE_PATH = File.join(PATH, '.cache')
- TMP_PATH = File.join(PATH, '.tmp')
-
- ##
# Autoload Backup base files
autoload :Model, File.join(LIBRARY_PATH, 'model')
autoload :Archive, File.join(LIBRARY_PATH, 'archive')
autoload :Packager, File.join(LIBRARY_PATH, 'packager')
+ autoload :Package, File.join(LIBRARY_PATH, 'package')
autoload :Cleaner, File.join(LIBRARY_PATH, 'cleaner')
autoload :Splitter, File.join(LIBRARY_PATH, 'splitter')
- autoload :Finder, File.join(LIBRARY_PATH, 'finder')
+ autoload :Config, File.join(LIBRARY_PATH, 'config')
autoload :Binder, File.join(LIBRARY_PATH, 'binder')
autoload :Template, File.join(LIBRARY_PATH, 'template')
autoload :Dependency, File.join(LIBRARY_PATH, 'dependency')
@@ -89,7 +63,7 @@ module CLI
# Autoload Backup storage files
module Storage
autoload :Base, File.join(STORAGE_PATH, 'base')
- autoload :Object, File.join(STORAGE_PATH, 'object')
+ autoload :Cycler, File.join(STORAGE_PATH, 'cycler')
autoload :S3, File.join(STORAGE_PATH, 's3')
autoload :CloudFiles, File.join(STORAGE_PATH, 'cloudfiles')
autoload :Ninefold, File.join(STORAGE_PATH, 'ninefold')
@@ -105,8 +79,13 @@ module Storage
# Autoload Backup syncer files
module Syncer
autoload :Base, File.join(SYNCER_PATH, 'base')
- autoload :RSync, File.join(SYNCER_PATH, 'rsync')
autoload :S3, File.join(SYNCER_PATH, 's3')
+ module RSync
+ autoload :Base, File.join(SYNCER_PATH, 'rsync', 'base')
+ autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
+ autoload :Push, File.join(SYNCER_PATH, 'rsync', 'push')
+ autoload :Pull, File.join(SYNCER_PATH, 'rsync', 'pull')
+ end
end
##
@@ -195,8 +174,14 @@ module Storage
end
module Syncer
- autoload :RSync, File.join(CONFIGURATION_PATH, 'syncer', 'rsync')
+ autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'base')
autoload :S3, File.join(CONFIGURATION_PATH, 'syncer', 's3')
+ module RSync
+ autoload :Base, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'base')
+ autoload :Local, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'local')
+ autoload :Push, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'push')
+ autoload :Pull, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'pull')
+ end
end
module Database
@@ -209,12 +194,4 @@ module Database
end
end
- ##
- # Dynamically defines all the available database, storage, compressor, encryptor and notifier
- # classes inside Backup::Finder to improve the DSL for the configuration file
- (DATABASES + STORAGES + COMPRESSORS + ENCRYPTORS + NOTIFIERS + SYNCERS).each do |constant|
- unless Backup::Finder.const_defined?(constant)
- Backup::Finder.const_set(constant, Class.new)
- end
- end
end
View
60 lib/backup/archive.rb
@@ -17,24 +17,34 @@ class Archive
attr_accessor :excludes
##
- # Stores the path to the archive directory
- attr_accessor :archive_path
+ # String of additional arguments for the `tar` command
+ attr_accessor :tar_args
##
# Takes the name of the archive and the configuration block
- def initialize(name, &block)
- @name = name.to_sym
+ def initialize(model, name, &block)
+ @model = model
+ @name = name.to_s
@paths = Array.new
@excludes = Array.new
- @tar_options = ''
+ @tar_args = ''
- instance_eval(&block)
+ instance_eval(&block) if block_given?
end
##
# Adds new paths to the @paths instance variable array
def add(path)
- @paths << File.expand_path(path)
+ path = File.expand_path(path)
+ if File.exist?(path)
+ @paths << path
+ else
+ Logger.warn Errors::Archive::NotFoundError.new(<<-EOS)
+ The following path was not found:
+ #{ path }
+ This path will be omitted from the '#{ name }' Archive.
+ EOS
+ end
end
##
@@ -47,39 +57,53 @@ def exclude(path)
# Adds the given String of +options+ to the `tar` command.
# e.g. '-h --xattrs'
def tar_options(options)
- @tar_options = options
+ @tar_args = options
end
##
# Archives all the provided paths in to a single .tar file
# and places that .tar file in the folder which later will be packaged
+ # If the model is configured with a Compressor, the tar command output
+ # will be piped through the Compressor command and the file extension
+ # will be adjusted to indicate the type of compression used.
def perform!
- @archive_path = File.join(TMP_PATH, TRIGGER, 'archive')
- mkdir(archive_path)
-
Logger.message "#{ self.class } started packaging and archiving:\n" +
paths.map {|path| " #{path}" }.join("\n")
- run("#{ utility(:tar) } #{ @tar_options } -cf '#{ File.join(archive_path, "#{name}.tar") }' " +
- "#{ paths_to_exclude } #{ paths_to_package }", :ignore_exit_codes => [1])
+ archive_path = File.join(Config.tmp_path, @model.trigger, 'archives')
+ FileUtils.mkdir_p(archive_path)
+
+ archive_ext = 'tar'
+ archive_cmd = "#{ utility(:tar) } #{ tar_args } -cf - " +
+ "#{ paths_to_exclude } #{ paths_to_package }"
+
+ if @model.compressor
+ @model.compressor.compress_with do |command, ext|
+ archive_cmd << " | #{command}"
+ archive_ext << ext
+ end
+ end
+
+ archive_cmd << " > '#{ File.join(archive_path, "#{name}.#{archive_ext}") }'"
+
+ run(archive_cmd)
end
- private
+ private
##
# Returns a "tar-ready" string of all the specified paths combined
def paths_to_package
- paths.map do |path|
- "'#{path}'"
- end.join("\s")
+ paths.map {|path| "'#{path}'" }.join(' ')
end
##
# Returns a "tar-ready" string of all the specified excludes combined
def paths_to_exclude
if excludes.any?
- excludes.map{ |e| "--exclude='#{e}'" }.join(" ")
+ excludes.map {|path| "--exclude='#{path}'" }.join(' ')
end
end
+
end
end
View
135 lib/backup/cleaner.rb
@@ -1,36 +1,121 @@
# encoding: utf-8
module Backup
- class Cleaner
- include Backup::CLI::Helpers
+ module Cleaner
+ class << self
- ##
- # Holds an instance of the current Backup model
- attr_accessor :model
+ ##
+ # Logs warnings if any temporary files still exist
+ # from the last time this model/trigger was run,
+ # then removes the files.
+ def prepare(model)
+ @model = model
- ##
- # Creates a new instance of Backup::Cleaner
- def initialize(model)
- @model = model
- end
+ messages = []
+ if packaging_folder_dirty?
+ messages << <<-EOS
+ The temporary backup folder still contains files!
+ '#{ File.join(Config.tmp_path, @model.trigger) }'
+ These files will now be removed.
+ EOS
+ FileUtils.rm_rf(File.join(Config.tmp_path, @model.trigger))
+ end
- ##
- # Runs the cleaner object which removes all the tmp files generated by Backup
- def clean!
- Logger.message "#{self.class} started cleaning up the temporary files."
- run("#{ utility(:rm) } -rf #{ paths.map { |path| "'#{ path }'" }.join(" ") }")
- end
+ package_files = tmp_path_package_files
+ unless package_files.empty?
+ # the chances that tmp_path would be dirty
+ # AND package files exist are practically nil
+ messages << ('-' * 74) unless messages.empty?
+
+ messages << <<-EOS
+ The temporary backup folder '#{ Config.tmp_path }'
+ appears to contain the package files from the previous backup!
+ #{ package_files.join("\n") }
+ These files will now be removed.
+ EOS
+ package_files.each {|file| FileUtils.rm_f(file) }
+ end
+
+ unless messages.empty?
+ Logger.warn Errors::CleanerError.new(<<-EOS)
+ Cleanup Warning
+ #{ messages.join("\n") }
+ Please check the log for messages and/or your notifications
+ concerning this backup: '#{ @model.label } (#{ @model.trigger })'
+ The temporary files which had to be removed should not have existed.
+ EOS
+ end
+ end
+
+ ##
+ # Remove the temporary folder used during packaging
+ def remove_packaging(model)
+ Logger.message "Cleaning up the temporary files..."
+ FileUtils.rm_rf(File.join(Config.tmp_path, model.trigger))
+ end
+
+ ##
+ # Remove the final package files from tmp_path
+ # Note: 'force' is used, since a Local Storage may *move* these files.
+ def remove_package(package)
+ Logger.message "Cleaning up the package files..."
+ package.filenames.each do |file|
+ FileUtils.rm_f(File.join(Config.tmp_path, file))
+ end
+ end
+
+ ##
+ # Logs warnings if any temporary files still exist
+ # when errors occur during the backup
+ def warnings(model)
+ @model = model
+
+ messages = []
+ if packaging_folder_dirty?
+ messages << <<-EOS
+ The temporary backup folder still contains files!
+ '#{ File.join(Config.tmp_path, @model.trigger) }'
+ This folder may contain completed Archives and/or Database backups.
+ EOS
+ end
+
+ package_files = tmp_path_package_files
+ unless package_files.empty?
+ # the chances that tmp_path would be dirty
+ # AND package files exist are practically nil
+ messages << ('-' * 74) unless messages.empty?
+
+ messages << <<-EOS
+ The temporary backup folder '#{ Config.tmp_path }'
+ appears to contain the backup files which were to be stored:
+ #{ package_files.join("\n") }
+ EOS
+ end
+
+ unless messages.empty?
+ Logger.warn Errors::CleanerError.new(<<-EOS)
+ Cleanup Warning
+ #{ messages.join("\n") }
+ Make sure you check these files before the next scheduled backup for
+ '#{ @model.label } (#{ @model.trigger })'
+ These files will be removed at that time!
+ EOS
+ end
+ end
+
+ private
+
+ def packaging_folder_dirty?
+ !Dir[File.join(Config.tmp_path, @model.trigger, '*')].empty?
+ end
- private
+ def tmp_path_package_files
+ Dir[File.join(
+ Config.tmp_path,
+ "????.??.??.??.??.??.#{ @model.trigger }.tar{,[.-]*}"
+ )]
+ end
- ##
- # Returns an Array of paths to temporary files generated by Backup that need to be removed
- def paths
- Array.new([
- File.join(TMP_PATH, TRIGGER),
- Backup::Model.file,
- Backup::Model.chunk_suffixes.map { |chunk_suffix| "#{Backup::Model.file}-#{chunk_suffix}" }
- ]).flatten
end
end
end
View
49 lib/backup/cli/helpers.rb
@@ -3,6 +3,7 @@
module Backup
module CLI
module Helpers
+ UTILITY = {}
##
# Runs a given command in an isolated (sub) process using POpen4.
@@ -33,44 +34,28 @@ def run(command, options = {})
end
##
- # Wrapper method for FileUtils.mkdir_p to create directories
- # through a ruby method. This helps with test coverage and
- # improves readability
- def mkdir(path)
- FileUtils.mkdir_p(path)
- end
-
- ##
- # Wrapper for the FileUtils.rm_rf to remove files and folders
- # through a ruby method. This helps with test coverage and
- # improves readability
- def rm(path)
- FileUtils.rm_rf(path)
- end
-
- ##
- # Tries to find the full path of the specified utility. If the full
- # path is found, it'll return that. Otherwise it'll just return the
- # name of the utility. If the 'utility_path' is defined, it'll check
- # to see if it isn't an empty string, and if it isn't, it'll go ahead and
- # always use that path rather than auto-detecting it
+ # Returns the full path to the specified utility.
+ # Raises an error if utility can not be found in the system's $PATH
def utility(name)
- if respond_to?(:utility_path)
- if utility_path.is_a?(String) and not utility_path.empty?
- return utility_path
- end
- end
-
- if path = %x[which #{name} 2>/dev/null].chomp and not path.empty?
- return path
+ path = UTILITY[name] || %x[which #{name} 2>/dev/null].chomp
+ if path.empty?
+ raise Errors::CLI::UtilityNotFoundError, <<-EOS
+ Path to '#{ name }' could not be found.
+ Make sure the specified utility is installed
+ and available in your system's $PATH.
+ If this is a database utility, you may need to specify the full path
+ using the Database's '<utility_name>_utility' configuration setting.
+ EOS
end
- name
+ UTILITY[name] = path
end
##
- # Returns the name of the command
+ # Returns the name of the command name from the given command line
def command_name(command)
- command.slice(0, command.index(/\s/)).split('/')[-1]
+ i = command =~ /\s/
+ command = command.slice(0, i) if i
+ command.split('/')[-1]
end
##
View
153 lib/backup/cli/utility.rb
@@ -13,7 +13,7 @@ class Utility < Thor
# If the other options (--config-file, --data-path, --cache--path, --tmp-path) aren't specified
# they will fallback to the (good) defaults
#
- # If --root-path is given, it will be used as the base (Backup::PATH) for our defaults,
+ # If --root-path is given, it will be used as the base path for our defaults,
# as well as the base path for any option specified as a relative path.
# Any option given as an absolute path will be used "as-is".
method_option :trigger, :type => :string, :required => true, :aliases => ['-t', '--triggers']
@@ -30,71 +30,56 @@ class Utility < Thor
"This will invoke 4 backups, and they will run in the order specified (not asynchronous)."
def perform
##
- # Setup required paths based on the given options
- setup_paths(options)
+ # Silence Backup::Logger from printing to STDOUT, if --quiet was specified
+ Logger.quiet = options[:quiet]
##
- # Silence Backup::Logger from printing to STDOUT, if --quiet was specified
- Logger.send(:const_set, :QUIET, options[:quiet])
+ # Update Config variables based on the given options
+ Config.update(options)
+
+ ##
+ # Load the configuration file
+ Config.load_config!
+
+ ##
+ # Ensure the :log_path, :cache_path and :tmp_path are created
+ # if they do not yet exist
+ [Config.log_path, Config.cache_path, Config.tmp_path].each do |path|
+ FileUtils.mkdir_p(path)
+ end
+
+ ##
+ # Truncate log file if needed
+ Logger.truncate!
##
# Prepare all trigger names by splitting them by ','
# and finding trigger names matching wildcard
triggers = options[:trigger].split(",")
- triggers.map!(&:strip).map!{ |t|
- t.include?(Backup::Finder::WILDCARD) ?
- Backup::Finder.new(t).matching : t
+ triggers.map!(&:strip).map! {|t|
+ t.include?('*') ? Model.find_matching(t) : t
}.flatten!
##
# Process each trigger
triggers.each do |trigger|
-
- ##
- # Defines the TRIGGER constant
- Backup.send(:const_set, :TRIGGER, trigger)
-
##
- # Define the TIME constants
- Backup.send(:const_set, :TIME, Time.now.strftime("%Y.%m.%d.%H.%M.%S"))
+ # Find the model for this trigger
+ # Will raise an error if not found
+ model = Model.find(trigger)
##
- # Ensure DATA_PATH and DATA_PATH/TRIGGER are created if they do not yet exist
- FileUtils.mkdir_p(File.join(Backup::DATA_PATH, Backup::TRIGGER))
-
- ##
- # Parses the backup configuration file and returns the model instance by trigger
- model = Backup::Finder.new(trigger).find
-
- ##
- # Runs the returned model
- Logger.message "Performing backup for #{model.label}!"
+ # Prepare and Perform the backup
+ model.prepare!
model.perform!
##
- # Removes the TRIGGER constant
- Backup.send(:remove_const, :TRIGGER) if defined? Backup::TRIGGER
-
- ##
- # Removes the TIME constant
- Backup.send(:remove_const, :TIME) if defined? Backup::TIME
-
- ##
- # Reset the Backup::Model.current to nil for the next potential run
- Backup::Model.current = nil
-
- ##
# Clear the Log Messages for the next potential run
Logger.clear!
-
- ##
- # Reset the Backup::Model.extension to 'tar' so it's at its
- # initial state when the next Backup::Model initializes
- Backup::Model.extension = 'tar'
end
rescue => err
- Logger.error Backup::Errors::CLIError.wrap(err)
+ Logger.error Errors::CLIError.wrap(err)
exit(1)
end
@@ -104,14 +89,15 @@ def perform
# For example:
# $ backup generate:model --trigger my_backup --databases='mongodb'
# will generate a pre-populated model with a base MongoDB setup
- method_option :trigger, :type => :string, :required => true
- method_option :config_path, :type => :string,
- :desc => 'Path to your Backup configuration directory'
desc 'generate:model', "Generates a Backup model file\n\n" +
"Note:\n" +
"\s\s'--config-path' is the path to the directory where 'config.rb' is located.\n" +
"\s\sThe model file will be created as '<config_path>/models/<trigger>.rb'\n" +
- "\s\sDefault: #{Backup::PATH}\n"
+ "\s\sDefault: #{Config.root_path}\n"
+
+ method_option :trigger, :type => :string, :required => true
+ method_option :config_path, :type => :string,
+ :desc => 'Path to your Backup configuration directory'
# options with their available values
%w{ databases storages syncers
@@ -127,10 +113,11 @@ def perform
define_method "generate:model" do
opts = options.merge(
- :trigger => options[:trigger].gsub(/[\W\s]/, '_'),
- :config_path => options[:config_path] ? File.expand_path(options[:config_path]) : nil
+ :trigger => options[:trigger].gsub(/[\W\s]/, '_'),
+ :config_path => options[:config_path] ?
+ File.expand_path(options[:config_path]) : nil
)
- config_path = opts[:config_path] || Backup::PATH
+ config_path = opts[:config_path] || Config.root_path
models_path = File.join(config_path, "models")
config = File.join(config_path, "config.rb")
model = File.join(models_path, "#{opts[:trigger]}.rb")
@@ -141,14 +128,14 @@ def perform
file.write(Backup::Template.new({:options => opts}).
result("cli/utility/model.erb"))
end
- puts "Generated model file in '#{ model }'."
+ puts "Generated model file: '#{ model }'."
end
if not File.exist?(config)
File.open(config, "w") do |file|
file.write(Backup::Template.new.result("cli/utility/config"))
end
- puts "Generated configuration file in '#{ config }'."
+ puts "Generated configuration file: '#{ config }'."
end
end
@@ -156,18 +143,19 @@ def perform
# [Generate:Config]
# Generates the main configuration file
desc 'generate:config', 'Generates the main Backup bootstrap/configuration file'
- method_option :path, :type => :string
+ method_option :config_path, :type => :string,
+ :desc => 'Path to your Backup configuration directory'
define_method 'generate:config' do
- path = options[:path] ? File.expand_path(options[:path]) : nil
- config_path = path || Backup::PATH
- config = File.join(config_path, "config.rb")
+ config_path = options[:config_path] ?
+ File.expand_path(options[:config_path]) : Config.root_path
+ config = File.join(config_path, "config.rb")
FileUtils.mkdir_p(config_path)
if overwrite?(config)
File.open(config, "w") do |file|
file.write(Backup::Template.new.result("cli/utility/config"))
end
- puts "Generated configuration file in '#{ config }'"
+ puts "Generated configuration file: '#{ config }'."
end
end
@@ -192,7 +180,7 @@ def decrypt
%x[gpg -o '#{options[:out]}' -d '#{options[:in]}']
else
puts "Unknown encryptor: #{options[:encryptor]}"
- puts "Use either 'openssl' or 'gpg'"
+ puts "Use either 'openssl' or 'gpg'."
end
end
@@ -246,55 +234,6 @@ def version
private
##
- # Setup required paths based on the given options
- #
- def setup_paths(options)
- ##
- # Set PATH if --root-path is given and the directory exists
- root_path = false
- root_given = options[:root_path].strip
- if !root_given.empty? && File.directory?(root_given)
- root_path = File.expand_path(root_given)
- Backup.send(:remove_const, :PATH)
- Backup.send(:const_set, :PATH, root_path)
- end
-
- ##
- # Update all defaults and given paths to use root_path (if given).
- # Paths given as an absolute path will be used 'as-is'
- { :config_file => 'config.rb',
- :data_path => 'data',
- :log_path => 'log',
- :cache_path => '.cache',
- :tmp_path => '.tmp' }.each do |key, name|
- # strip any trailing '/' in case the user supplied this as part of
- # an absolute path, so we can match it against File.expand_path()
- given = options[key].sub(/\/\s*$/, '').lstrip
- path = false
- if given.empty?
- path = File.join(root_path, name) if root_path
- else
- path = File.expand_path(given)
- unless given == path
- path = File.join(root_path, given) if root_path
- end
- end
-
- const = key.to_s.upcase
- if path
- Backup.send(:remove_const, const)
- Backup.send(:const_set, const, path)
- else
- path = Backup.const_get(const)
- end
-
- ##
- # Ensure the LOG_PATH, CACHE_PATH and TMP_PATH are created if they do not yet exist
- FileUtils.mkdir_p(path) if [:log_path, :cache_path, :tmp_path].include?(key)
- end
- end
-
- ##
# Helper method for asking the user if he/she wants to overwrite the file
def overwrite?(path)
if File.exist?(path)
View
16 lib/backup/compressor/base.rb
@@ -6,11 +6,23 @@ class Base
include Backup::CLI::Helpers
include Backup::Configuration::Helpers
+ def initialize
+ load_defaults!
+ end
+
+ private
+
+ ##
+ # Return the encryptor name, with Backup namespace removed
+ def compressor_name
+ self.class.to_s.sub('Backup::', '')
+ end
+
##
# Logs a message to the console and log file to inform
- # the client that Backup is compressing the archive
+ # the client that Backup is using the compressor
def log!
- Logger.message "#{ self.class } started compressing the archive."
+ Logger.message "Using #{ compressor_name } for compression."
end
end
end
View
34 lib/backup/compressor/bzip2.rb
@@ -7,12 +7,12 @@ class Bzip2 < Base
##
# Tells Backup::Compressor::Bzip2 to compress
# better (-9) rather than faster when set to true
- attr_writer :best
+ attr_accessor :best
##
# Tells Backup::Compressor::Bzip2 to compress
# faster (-1) rather than better when set to true
- attr_writer :fast
+ attr_accessor :fast
##
# Creates a new instance of Backup::Compressor::Bzip2 and
@@ -21,7 +21,7 @@ class Bzip2 < Base
# and lower block sizes don't make things significantly faster
# (according to official bzip2 docs)
def initialize(&block)
- load_defaults!
+ super
@best ||= false
@fast ||= false
@@ -30,33 +30,19 @@ def initialize(&block)
end
##
- # Performs the compression of the packages backup file
- def perform!
+ # Yields to the block the compressor command with options
+ # and it's filename extension.
+ def compress_with
log!
- run("#{ utility(:bzip2) } #{ options } '#{ Backup::Model.file }'")
- Backup::Model.extension += '.bz2'
+ yield "#{ utility(:bzip2) }#{ options }", '.bz2'
end
- private
+ private
##
- # Combines the provided options and returns a bzip2 options string
+ # Returns the option syntax for compressing
def options
- (best + fast).join("\s")
- end
-
- ##
- # Returns the bzip2 option syntax for compressing
- # setting @best to true is redundant, as bzip2 compresses best by default
- def best
- return ['--best'] if @best; []
- end
-
- ##
- # Returns the bzip2 option syntax for compressing
- # (not significantly) faster when @fast is set to true
- def fast
- return ['--fast'] if @fast; []
+ " #{ '--best ' if @best }#{ '--fast' if @fast }".rstrip
end
end
View
34 lib/backup/compressor/gzip.rb
@@ -7,18 +7,18 @@ class Gzip < Base
##
# Tells Backup::Compressor::Gzip to compress
# better rather than faster when set to true
- attr_writer :best
+ attr_accessor :best
##
# Tells Backup::Compressor::Gzip to compress
# faster rather than better when set to true
- attr_writer :fast
+ attr_accessor :fast
##
# Creates a new instance of Backup::Compressor::Gzip and
# configures it to either compress faster or better
def initialize(&block)
- load_defaults!
+ super
@best ||= false
@fast ||= false
@@ -27,33 +27,19 @@ def initialize(&block)
end
##
- # Performs the compression of the packages backup file
- def perform!
+ # Yields to the block the compressor command with options
+ # and it's filename extension.
+ def compress_with
log!
- run("#{ utility(:gzip) } #{ options } '#{ Backup::Model.file }'")
- Backup::Model.extension += '.gz'
+ yield "#{ utility(:gzip) }#{ options }", '.gz'
end
- private
-
- ##
- # Combines the provided options and returns a gzip options string
- def options
- (best + fast).join("\s")
- end
+ private
##
# Returns the gzip option syntax for compressing
- # better when @best is set to true
- def best
- return ['--best'] if @best; []
- end
-
- ##
- # Returns the gzip option syntax for compressing
- # faster when @fast is set to true
- def fast
- return ['--fast'] if @fast; []
+ def options
+ " #{ '--best ' if @best }#{ '--fast' if @fast }".rstrip
end
end
View
33 lib/backup/compressor/lzma.rb
@@ -7,12 +7,12 @@ class Lzma < Base
##
# Tells Backup::Compressor::Lzma to compress
# better (-9) rather than faster when set to true
- attr_writer :best
+ attr_accessor :best
##
# Tells Backup::Compressor::Lzma to compress
# faster (-1) rather than better when set to true
- attr_writer :fast
+ attr_accessor :fast
##
# Creates a new instance of Backup::Compressor::Lzma and
@@ -21,7 +21,7 @@ class Lzma < Base
# and lower block sizes don't make things significantly faster
# (according to official bzip2 docs)
def initialize(&block)
- load_defaults!
+ super
@best ||= false
@fast ||= false
@@ -30,34 +30,21 @@ def initialize(&block)
end
##
- # Performs the compression of the packages backup file
- def perform!
+ # Yields to the block the compressor command with options
+ # and it's filename extension.
+ def compress_with
log!
- run("#{ utility(:lzma) } #{ options } '#{ Backup::Model.file }'")
- Backup::Model.extension += '.lzma'
+ yield "#{ utility(:lzma) }#{ options }", '.lzma'
end
- private
+ private
##
- # Combines the provided options and returns a bzip2 options string
+ # Returns the option syntax for compressing
def options
- (best + fast).join("\s")
+ " #{ '--best ' if @best }#{ '--fast' if @fast }".rstrip
end
- ##
- # Returns the lzma option syntax for compressing
- # setting @best to true is redundant, as lzma compresses best by default
- def best
- return ['--best'] if @best; []
- end
-
- ##
- # Returns the lzma option syntax for compressing
- # (not significantly) faster when @fast is set to true
- def fast
- return ['--fast'] if @fast; []
- end
end
end
end
View
44 lib/backup/compressor/pbzip2.rb
@@ -7,17 +7,17 @@ class Pbzip2 < Base
##
# Tells Backup::Compressor::Pbzip2 to compress
# better (-9) rather than faster when set to true
- attr_writer :best
+ attr_accessor :best
##
# Tells Backup::Compressor::Pbzip2 to compress
# faster (-1) rather than better when set to true
- attr_writer :fast
+ attr_accessor :fast
##
# Tells Backup::Compressor::Pbzip2 how many processors
# use, by default autodetect is used
- attr_writer :processors
+ attr_accessor :processors
##
# Creates a new instance of Backup::Compressor::Pbzip2 and
@@ -26,51 +26,31 @@ class Pbzip2 < Base
# and lower block sizes don't make things significantly faster
# (according to official bzip2 docs)
def initialize(&block)
- load_defaults!
+ super
@best ||= false
@fast ||= false
- @processors ||= nil
+ @processors ||= false
instance_eval(&block) if block_given?
end
##
- # Performs the compression of the packages backup file
- def perform!
+ # Yields to the block the compressor command with options
+ # and it's filename extension.
+ def compress_with
log!
- run("#{ utility(:pbzip2) } #{ options } '#{ Backup::Model.file }'")
- Backup::Model.extension += '.bz2'
+ yield "#{ utility(:pbzip2) }#{ options }", '.bz2'
end
- private
+ private
##
- # Combines the provided options and returns a pbzip2 options string
+ # Returns the gzip option syntax for compressing
def options
- (best + fast + processors).join("\s")
+ " #{ '--best ' if @best }#{ '--fast ' if @fast }#{ "-p#{@processors}" if @processors }".rstrip
end
- ##
- # Returns the pbzip2 option syntax for compressing
- # setting @best to true is redundant, as pbzip2 compresses best by default
- def best
- return ['--best'] if @best; []
- end
-
- ##
- # Returns the pbzip2 option syntax for compressing
- # (not significantly) faster when @fast is set to true
- def fast
- return ['--fast'] if @fast; []
- end
-
- ##
- # Returns the pbzip2 option syntax for compressing
- # using given count of cpus
- def processors
- return ['-p' + @processors.to_s] if @processors; []
- end
end
end
end
View
171 lib/backup/config.rb
@@ -0,0 +1,171 @@
+# encoding: utf-8
+
+module Backup
+ module Config
+ DEFAULTS = {
+ :config_file => 'config.rb',
+ :data_path => 'data',
+ :log_path => 'log',
+ :cache_path => '.cache',
+ :tmp_path => '.tmp'
+ }
+
+ class << self
+ attr_reader :user, :root_path, :config_file,
+ :data_path, :log_path, :cache_path, :tmp_path
+
+ ##
+ # Setup required paths based on the given options
+ def update(options = {})
+ root_path = options[:root_path].to_s.strip
+ new_root = root_path.empty? ? false : set_root_path(root_path)
+
+ DEFAULTS.each do |name, ending|
+ set_path_variable(name, options[name], ending, new_root)
+ end
+ end
+
+ ##
+ # Tries to find and load the configuration file
+ def load_config!
+ unless File.exist?(@config_file)
+ raise Errors::Config::NotFoundError,
+ "Could not find configuration file: '#{@config_file}'."
+ end
+
+ module_eval(File.read(@config_file), @config_file)
+ end
+
+ private
+
+ ##
+ # Sets the @root_path to the given +path+ and returns it.
+ # Raises an error if the given +path+ does not exist.
+ def set_root_path(path)
+ # allows #reset! to set the default @root_path,
+ # then use #update to set all other paths,
+ # without requiring that @root_path exist.
+ return @root_path if path == @root_path
+
+ path = File.expand_path(path)
+ unless File.directory?(path)
+ raise Errors::Config::NotFoundError, <<-EOS
+ Root Path Not Found
+ When specifying a --root-path, the path must exist.
+ Path was: #{ path }
+ EOS
+ end
+ @root_path = path
+ end
+
+ def set_path_variable(name, path, ending, root_path)
+ # strip any trailing '/' in case the user supplied this as part of
+ # an absolute path, so we can match it against File.expand_path()
+ path = path.to_s.sub(/\/\s*$/, '').lstrip
+ new_path = false
+ if path.empty?
+ new_path = File.join(root_path, ending) if root_path
+ else
+ new_path = File.expand_path(path)
+ unless path == new_path
+ new_path = File.join(root_path, path) if root_path
+ end
+ end
+ instance_variable_set(:"@#{name}", new_path) if new_path
+ end
+
+ ##
+ # Set default values for accessors
+ def reset!
+ @user = ENV['USER'] || Etc.getpwuid.name
+ @root_path = File.join(File.expand_path(ENV['HOME'] || ''), 'Backup')
+ update(:root_path => @root_path)
+ end
+
+ ##
+ # List the available database, storage, syncer, compressor, encryptor
+ # and notifier constants. These are used to dynamically define these
+ # constant names inside Backup::Config to provide a nicer configuration
+ # file DSL syntax to the users. Adding existing constants to the arrays
+ # below will enable the user to use a constant instead of a string.
+ # Nested namespaces are represented using Hashs. Deep nesting supported.
+ #
+ # Example, instead of:
+ # database "MySQL" do |mysql|
+ # sync_with "RSync::Local" do |rsync|
+ #
+ # You can do:
+ # database MySQL do |mysql|
+ # sync_with RSync::Local do |rsync|
+ #
+ def add_dsl_constants!
+ create_modules(
+ self,
+ [ # Databases
+ ['MySQL', 'PostgreSQL', 'MongoDB', 'Redis', 'Riak'],
+ # Storages
+ ['S3', 'CloudFiles', 'Ninefold', 'Dropbox', 'FTP',
+ 'SFTP', 'SCP', 'RSync', 'Local'],
+ # Compressors
+ ['Gzip', 'Bzip2', 'Pbzip2', 'Lzma'],
+ # Encryptors
+ ['OpenSSL', 'GPG'],
+ # Syncers
+ ['S3', { 'RSync' => ['Push', 'Pull', 'Local'] }],
+ # Notifiers
+ ['Mail', 'Twitter', 'Campfire', 'Presently', 'Prowl', 'Hipchat']
+ ]
+ )
+ end
+
+ def create_modules(scope, names)
+ names.flatten.each do |name|
+ if name.is_a?(Hash)
+ name.each do |key, val|
+ create_modules(get_or_create_empty_module(scope, key), [val])
+ end
+ else
+ get_or_create_empty_module(scope, name)
+ end
+ end
+ end
+
+ def get_or_create_empty_module(scope, const)
+ if scope.const_defined?(const)
+ scope.const_get(const)
+ else
+ scope.const_set(const, Module.new)
+ end
+ end
+ end
+
+ ##
+ # Add the DSL constants and set default values for accessors when loaded.
+ add_dsl_constants!
+ reset!
+ end
+
+ ##
+ # Warn user of deprecated Backup::CONFIG_FILE constant reference
+ # in older config.rb files and return the proper Config.config_file value.
+ class << self
+ def const_missing(const)
+ if const.to_s == 'CONFIG_FILE'
+ Logger.warn Errors::ConfigError.new(<<-EOS)
+ Configuration File Upgrade Needed
+ Your configuration file, located at #{ Config.config_file }
+ needs to be upgraded for this version of Backup.
+ The reference to 'Backup::CONFIG_FILE' in your current config.rb file
+ has been deprecated and needs to be replaced with 'Config.config_file'.
+ You may update this reference in your config.rb manually,
+ or generate a new config.rb using 'backup generate:config'.
+ * Note: if you have global configuration defaults set in config.rb,
+ be sure to transfer them to your new config.rb, should you choose
+ to generate a new config.rb file.
+ EOS
+ return Config.config_file
+ end
+ super
+ end
+ end
+end
View
3  lib/backup/configuration/compressor/base.rb
@@ -3,8 +3,7 @@
module Backup
module Configuration
module Compressor
- class Base < Backup::Configuration::Base
- end
+ class Base < Configuration::Base; end
end
end
end
View
8 lib/backup/configuration/compressor/pbzip2.rb
@@ -16,10 +16,10 @@ class << self
# faster (-1) (but not significantly faster)
attr_accessor :fast
- ##
- # Tells Backup::Compressor::Pbzip2 how many processors
- # use, by default autodetect is used
- attr_writer :processors
+ ##
+ # Tells Backup::Compressor::Pbzip2 how many processors
+ # use, by default autodetect is used
+ attr_accessor :processors
end
end
View
3  lib/backup/configuration/database/base.rb
@@ -3,12 +3,13 @@
module Backup
module Configuration
module Database
- class Base < Backup::Configuration::Base
+ class Base < Configuration::Base
class << self
##
# Allows the user to specify the path to a "dump" utility
# in case it cannot be auto-detected by Backup
+ # [DEPRECATED] - use <utility_name>_utility methods
attr_accessor :utility_path
end
View
8 lib/backup/configuration/database/mongodb.rb
@@ -31,6 +31,14 @@ class << self
attr_accessor :additional_options
##
+ # Path to the mongodump utility (optional)
+ attr_accessor :mongodump_utility
+
+ ##
+ # Path to the mongo utility (optional)
+ attr_accessor :mongo_utility
+
+ ##
# 'lock' dump meaning wrapping mongodump with fsync & lock
attr_accessor :lock
View
4 lib/backup/configuration/database/mysql.rb
@@ -30,6 +30,10 @@ class << self
# Additional "mysqldump" options
attr_accessor :additional_options
+ ##
+ # Path to mysqldump utility (optional)
+ attr_accessor :mysqldump_utility
+
end
end
end
View
4 lib/backup/configuration/database/postgresql.rb
@@ -30,6 +30,10 @@ class << self
# Additional "pg_dump" options
attr_accessor :additional_options
+ ##
+ # Path to pg_dump utility (optional)
+ attr_accessor :pg_dump_utility
+
end
end
end
View
4 lib/backup/configuration/database/redis.rb
@@ -28,6 +28,10 @@ class << self
# Additional "redis-cli" options
attr_accessor :additional_options
+ ##
+ # Path to the redis-cli utility (optional)
+ attr_accessor :redis_cli_utility
+
end
end
end
View
6 lib/backup/configuration/database/riak.rb
@@ -18,8 +18,12 @@ class << self
# Cookie is the Erlang cookie/shared secret used to connect to the node.
attr_accessor :cookie
+ ##
+ # Path to riak-admin utility (optional)
+ attr_accessor :riak_admin_utility
+
end
end
end
end
-end
+end
View
3  lib/backup/configuration/encryptor/base.rb
@@ -3,8 +3,7 @@
module Backup
module Configuration
module Encryptor
- class Base < Backup::Configuration::Base
- end
+ class Base < Configuration::Base; end
end
end
end
View
2  lib/backup/configuration/encryptor/open_ssl.rb
@@ -10,7 +10,7 @@ class << self
# The password that'll be used to encrypt the backup. This
# password will be required to decrypt the backup later on.
attr_accessor :password
-
+
##
# The password file used for encrypting the backup. This
# password file will be required to decrypt the backup later
View
9 lib/backup/configuration/helpers.rb
@@ -9,8 +9,11 @@ module Helpers
# configuration for these methods, if they respond then they will
# assign the object's attribute(s) to that particular global configuration's attribute
def load_defaults!
- c = self.class.name.split('::')
- configuration = Backup::Configuration.const_get(c[1]).const_get(c[2])
+ module_names = self.class.name.split('::')[1..-1]
+ configuration = Backup::Configuration
+ module_names.each do |module_name|
+ configuration = configuration.const_get(module_name)
+ end
getter_methods.each do |attribute|
if configuration.respond_to?(attribute)
@@ -27,6 +30,8 @@ def clear_defaults!
end
end
+ private
+
##
# Returns an Array of the setter methods (as String)
def setter_methods
View
32 lib/backup/configuration/notifier/base.rb
@@ -3,49 +3,25 @@
module Backup
module Configuration
module Notifier
- class Base < Backup::Configuration::Base
+ class Base < Configuration::Base
class << self
##
# When set to true, the user will be notified by email
# when a backup process ends without raising any exceptions
- attr_writer :on_success
+ attr_accessor :on_success
##
# When set to true, the user will be notified by email
# when a backup process ends successfully, but logged warnings
- attr_writer :on_warning
+ attr_accessor :on_warning
##
# When set to true, the user will be notified by email
# when a backup process raises an exception before finishing
- attr_writer :on_failure
+ attr_accessor :on_failure
end
-
- ##
- # When @on_success is nil it means it hasn't been defined
- # and will then default to true
- def self.on_success
- return true if @on_success.nil?
- @on_success
- end
-
- ##
- # When @on_success is nil it means it hasn't been defined
- # and will then default to true
- def self.on_warning
- return true if @on_warning.nil?
- @on_warning
- end
-
- ##
- # When @on_failure is nil it means it hasn't been defined
- # and will then default to true
- def self.on_failure
- return true if @on_failure.nil?
- @on_failure
- end
end
end
end
View
2  lib/backup/configuration/storage/base.rb
@@ -3,7 +3,7 @@
module Backup
module Configuration
module Storage
- class Base < Backup::Configuration::Base
+ class Base < Configuration::Base
class << self
##
View
18 lib/backup/configuration/storage/dropbox.rb
@@ -11,16 +11,26 @@ class << self
attr_accessor :api_key, :api_secret
##
- # Path to where the backups will be stored
- attr_accessor :path
+ # Dropbox Access Type
+ # Valid values are:
+ # :app_folder (default)
+ # :dropbox (full access)
+ attr_accessor :access_type
##
- # Dropbox connection timeout
- attr_accessor :timeout
+ # Path to where the backups will be stored
+ attr_accessor :path
# DEPRECATED METHODS #############################################
+ # Deprecated as of v3.0.21 - for move to official 'dropbox-sdk' gem (v1.1)
+ attr_reader :timeout
+ def timeout=(value)
+ Logger.warn "[DEPRECATED] Backup::Configuration::Storage::Dropbox.timeout=\n" +
+ " is deprecated and will be removed at some point."
+ end
+
def email
Logger.warn "[DEPRECATED] Backup::Configuration::Storage::Dropbox.email\n" +
" is deprecated and will be removed at some point."
View
10 lib/backup/configuration/syncer/base.rb
@@ -0,0 +1,10 @@
+# encoding: utf-8
+
+module Backup
+ module Configuration
+ module Syncer
+ class Base < Configuration::Base; end
+ end
+ end
+end
+
View
45 lib/backup/configuration/syncer/rsync.rb
@@ -1,45 +0,0 @@
-# encoding: utf-8
-
-module Backup
- module Configuration
- module Syncer
- class RSync < Base
- class << self
-
- ##
- # Server credentials
- attr_accessor :username, :password
-
- ##
- # Server IP Address and SSH port
- attr_accessor :ip
-
- ##
- # The SSH port to connect to
- attr_accessor :port
-
- ##
- # Directories to sync
- attr_accessor :directories
-
- ##
- # Path to store the synced files/directories to
- attr_accessor :path
-
- ##
- # Flag for mirroring the files/directories
- attr_accessor :mirror
-
- ##
- # Flag for compressing (only compresses for the transfer)
- attr_accessor :compress
-
- ##
- # Additional options for the rsync cli
- attr_accessor :additional_options
-
- end
- end
- end
- end
-end
View
28 lib/backup/configuration/syncer/rsync/base.rb
@@ -0,0 +1,28 @@
+# encoding: utf-8
+
+module Backup
+ module Configuration
+ module Syncer
+ module RSync