Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Initial Import

commit 370ff3cad64b65e8029488e7de32a10891611aec 1 parent 3f49782
derks authored
Showing with 33,138 additions and 1 deletion.
  1. +3 −0  .gitignore
  2. +235 −0 ChangeLog
  3. +57 −0 INSTALL
  4. +0 −1  README
  5. 0  addons/ACTIVE
  6. +72 −0 config/README
  7. +35 −0 config/backupsets/default.conf
  8. +13 −0 config/backupsets/examples/example.conf
  9. +95 −0 config/backupsets/examples/maatkit.conf
  10. +71 −0 config/backupsets/examples/mysql-lvm.conf
  11. +127 −0 config/backupsets/examples/mysqldump.conf
  12. +85 −0 config/backupsets/examples/mysqlhotcopy.conf
  13. +27 −0 config/holland.conf
  14. +5 −0 config/providers/README
  15. +3 −0  config/providers/example.conf
  16. +77 −0 config/providers/maatkit.conf
  17. +49 −0 config/providers/mysql-lvm.conf
  18. +124 −0 config/providers/mysqldump.conf
  19. +73 −0 config/providers/mysqlhotcopy.conf
  20. +29 −0 devtools/README
  21. +1,482 −0 devtools/_virtualenv.py
  22. +118 −0 devtools/build_debs.py
  23. +34 −0 devtools/build_holland_doc.sh
  24. +116 −0 devtools/build_rpms.py
  25. +6 −0 devtools/charset_test.py
  26. +3 −0  devtools/lib/README
  27. +23 −0 devtools/lib/coverage-2.85/PKG-INFO
  28. +7 −0 devtools/lib/coverage-2.85/README.txt
  29. +1,163 −0 devtools/lib/coverage-2.85/coverage.py
  30. +5 −0 devtools/lib/coverage-2.85/setup.cfg
  31. +48 −0 devtools/lib/coverage-2.85/setup.py
  32. +259 −0 devtools/lib/mocker-0.10.1/LICENSE
  33. +109 −0 devtools/lib/mocker-0.10.1/NEWS
  34. +2,068 −0 devtools/lib/mocker-0.10.1/mocker.py
  35. +5 −0 devtools/lib/mocker-0.10.1/setup.cfg
  36. +36 −0 devtools/lib/mocker-0.10.1/setup.py
  37. +3,962 −0 devtools/lib/mocker-0.10.1/test.py
  38. +12 −0 devtools/load_it.py
  39. +189 −0 devtools/mkvirtenv.py
  40. +144 −0 devtools/mkvirtenv.sh
  41. +93 −0 devtools/python_vimrc
  42. +210 −0 devtools/test_holland.py
  43. +60 −0 devtools/test_it.sh
  44. +5 −0 distribution/debian/changelog
  45. +1 −0  distribution/debian/compat
  46. +75 −0 distribution/debian/control
  47. +23 −0 distribution/debian/copyright
  48. +2 −0  distribution/debian/holland-common.files
  49. +1 −0  distribution/debian/holland-commvault.dirs
  50. +4 −0 distribution/debian/holland-commvault.files
  51. +1 −0  distribution/debian/holland-commvault.lintian-overrides
  52. +1 −0  distribution/debian/holland-commvault.manpages
  53. +3 −0  distribution/debian/holland-example.files
  54. +3 −0  distribution/debian/holland-maatkit.files
  55. +1 −0  distribution/debian/holland-mysqlcmds.files
  56. +3 −0  distribution/debian/holland-mysqldump.files
  57. +3 −0  distribution/debian/holland-mysqlhotcopy.files
  58. +4 −0 distribution/debian/holland.dirs
  59. +8 −0 distribution/debian/holland.files
  60. +1 −0  distribution/debian/holland.manpages
  61. +1 −0  distribution/debian/pyversions
  62. +100 −0 distribution/debian/rules
  63. +327 −0 distribution/holland.spec
  64. +1 −0  distribution/ubuntu/ubuntu.jaunty/activeaddons
  65. +7 −0 distribution/ubuntu/ubuntu.jaunty/activeplugins
  66. +25 −0 distribution/ubuntu/ubuntu.jaunty/changelog
  67. +1 −0  distribution/ubuntu/ubuntu.jaunty/compat
  68. +73 −0 distribution/ubuntu/ubuntu.jaunty/control
  69. +23 −0 distribution/ubuntu/ubuntu.jaunty/copyright
  70. +7 −0 distribution/ubuntu/ubuntu.jaunty/dirs
  71. 0  distribution/ubuntu/ubuntu.jaunty/docs
  72. +126 −0 distribution/ubuntu/ubuntu.jaunty/rules
  73. +82 −0 docs/Makefile
  74. +89 −0 docs/devdocs/Makefile
  75. +288 −0 docs/devdocs/generate_modules.py
  76. +17 −0 docs/devdocs/initdevdocs.sh
  77. +113 −0 docs/devdocs/make.bat
  78. +13 −0 docs/devdocs/mkdevdocs.sh
  79. +206 −0 docs/devdocs/source/conf.py
  80. +61 −0 docs/devdocs/source/holland.core/holland.commands.rst
  81. +29 −0 docs/devdocs/source/holland.core/holland.core.backports.logging.rst
  82. +36 −0 docs/devdocs/source/holland.core/holland.core.backports.rst
  83. +21 −0 docs/devdocs/source/holland.core/holland.core.command.rst
  84. +37 −0 docs/devdocs/source/holland.core/holland.core.config.rst
  85. +63 −0 docs/devdocs/source/holland.core/holland.core.rst
  86. +45 −0 docs/devdocs/source/holland.core/holland.core.util.rst
  87. +13 −0 docs/devdocs/source/holland.core/holland.hooks.rst
  88. +16 −0 docs/devdocs/source/holland.core/holland.rst
  89. +14 −0 docs/devdocs/source/holland.core/modules.rst
  90. +22 −0 docs/devdocs/source/index.rst
  91. +13 −0 docs/devdocs/source/plugins/holland.backup.example/holland.backup.rst
  92. +12 −0 docs/devdocs/source/plugins/holland.backup.example/holland.rst
  93. +13 −0 docs/devdocs/source/plugins/holland.backup.example/modules.rst
  94. +13 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.actions.archive.rst
  95. +21 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.actions.mysql.rst
  96. +13 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.actions.rst
  97. +37 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.cli.rst
  98. +13 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.py23.rst
  99. +29 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.pylvm.rst
  100. +48 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.rst
  101. +5 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.tests.subscriber.mysql.rst
  102. +12 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.tests.subscriber.rst
  103. +21 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.lvm.util.rst
  104. +12 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.backup.rst
  105. +13 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.restore.rst
  106. +13 −0 docs/devdocs/source/plugins/holland.backup.lvm/holland.rst
  107. +15 −0 docs/devdocs/source/plugins/holland.backup.lvm/modules.rst
  108. +61 −0 docs/devdocs/source/plugins/holland.backup.lvm/tests.pylvm.rst
  109. +44 −0 docs/devdocs/source/plugins/holland.backup.lvm/tests.rst
  110. +13 −0 docs/devdocs/source/plugins/holland.backup.maatkit/holland.backup.rst
  111. +12 −0 docs/devdocs/source/plugins/holland.backup.maatkit/holland.rst
  112. +13 −0 docs/devdocs/source/plugins/holland.backup.maatkit/modules.rst
  113. +21 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/holland.backup.rst
  114. +21 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/holland.restore.mysqlrestore.mysqldump.rst
  115. +13 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/holland.restore.mysqlrestore.py23.rst
  116. +38 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/holland.restore.mysqlrestore.rst
  117. +20 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/holland.restore.rst
  118. +13 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/holland.rst
  119. +14 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/modules.rst
  120. +13 −0 docs/devdocs/source/plugins/holland.backup.mysqldump/tests.rst
  121. +13 −0 docs/devdocs/source/plugins/holland.backup.mysqlhotcopy/holland.backup.rst
  122. +12 −0 docs/devdocs/source/plugins/holland.backup.mysqlhotcopy/holland.rst
  123. +14 −0 docs/devdocs/source/plugins/holland.backup.mysqlhotcopy/modules.rst
  124. +13 −0 docs/devdocs/source/plugins/holland.backup.mysqlhotcopy/tests.rst
  125. +45 −0 docs/devdocs/source/plugins/holland.commands.mysql/holland.commands.rst
  126. +12 −0 docs/devdocs/source/plugins/holland.commands.mysql/holland.rst
  127. +13 −0 docs/devdocs/source/plugins/holland.commands.mysql/modules.rst
  128. +37 −0 docs/devdocs/source/plugins/holland.lib.common/holland.lib.archive.rst
  129. +36 −0 docs/devdocs/source/plugins/holland.lib.common/holland.lib.rst
  130. +12 −0 docs/devdocs/source/plugins/holland.lib.common/holland.rst
  131. +13 −0 docs/devdocs/source/plugins/holland.lib.common/modules.rst
  132. +37 −0 docs/devdocs/source/plugins/holland.lib.mysql/holland.lib.mysql.rst
  133. +12 −0 docs/devdocs/source/plugins/holland.lib.mysql/holland.lib.rst
  134. +12 −0 docs/devdocs/source/plugins/holland.lib.mysql/holland.rst
  135. +14 −0 docs/devdocs/source/plugins/holland.lib.mysql/modules.rst
  136. +29 −0 docs/devdocs/source/plugins/holland.lib.mysql/tests.rst
  137. +29 −0 docs/devdocs/source/plugins/index.rst
  138. BIN  docs/hollandaise_3.0_slides.key
  139. BIN  docs/hollandaise_3.pdf
  140. +4 −0 docs/man/README
  141. +195 −0 docs/man/holland.1
  142. +90 −0 docs/man/holland.rst
  143. +101 −0 docs/man/holland_cvmysqlsv.1
  144. +53 −0 docs/man/holland_cvmysqlsv.rst
  145. +22 −0 docs/source/api.rst
  146. +13 −0 docs/source/api/holland.backup.example.holland.backup.rst
  147. +12 −0 docs/source/api/holland.backup.example.holland.rst
  148. +13 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.actions.archive.rst
  149. +21 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.actions.mysql.rst
  150. +13 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.actions.rst
  151. +37 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.cli.rst
  152. +13 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.py23.rst
  153. +29 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.pylvm.rst
  154. +48 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.rst
  155. +21 −0 docs/source/api/holland.backup.lvm.holland.backup.lvm.util.rst
  156. +12 −0 docs/source/api/holland.backup.lvm.holland.backup.rst
  157. +13 −0 docs/source/api/holland.backup.lvm.holland.restore.rst
  158. +13 −0 docs/source/api/holland.backup.lvm.holland.rst
  159. +13 −0 docs/source/api/holland.backup.maatkit.holland.backup.rst
  160. +12 −0 docs/source/api/holland.backup.maatkit.holland.rst
  161. +21 −0 docs/source/api/holland.backup.mysqldump.holland.backup.rst
  162. +21 −0 docs/source/api/holland.backup.mysqldump.holland.restore.mysqlrestore.mysqldump.rst
  163. +13 −0 docs/source/api/holland.backup.mysqldump.holland.restore.mysqlrestore.py23.rst
  164. +38 −0 docs/source/api/holland.backup.mysqldump.holland.restore.mysqlrestore.rst
  165. +20 −0 docs/source/api/holland.backup.mysqldump.holland.restore.rst
  166. +13 −0 docs/source/api/holland.backup.mysqldump.holland.rst
  167. +13 −0 docs/source/api/holland.backup.mysqlhotcopy.holland.backup.rst
  168. +12 −0 docs/source/api/holland.backup.mysqlhotcopy.holland.rst
  169. +13 −0 docs/source/api/holland.backup.pgdump.holland.backup.rst
  170. +12 −0 docs/source/api/holland.backup.pgdump.holland.rst
  171. +13 −0 docs/source/api/holland.backup.xtrabackup.holland.backup.rst
  172. +12 −0 docs/source/api/holland.backup.xtrabackup.holland.rst
  173. +45 −0 docs/source/api/holland.commands.mysql.holland.commands.rst
  174. +12 −0 docs/source/api/holland.commands.mysql.holland.rst
  175. +61 −0 docs/source/api/holland.commands.rst
  176. +29 −0 docs/source/api/holland.core.backports.logging.rst
  177. +36 −0 docs/source/api/holland.core.backports.rst
  178. +21 −0 docs/source/api/holland.core.command.rst
  179. +37 −0 docs/source/api/holland.core.config.rst
  180. +63 −0 docs/source/api/holland.core.rst
  181. +45 −0 docs/source/api/holland.core.util.rst
  182. +13 −0 docs/source/api/holland.hooks.rst
  183. +37 −0 docs/source/api/holland.lib.common.holland.lib.archive.rst
  184. +36 −0 docs/source/api/holland.lib.common.holland.lib.rst
  185. +12 −0 docs/source/api/holland.lib.common.holland.rst
  186. +37 −0 docs/source/api/holland.lib.mysql.holland.lib.mysql.rst
  187. +12 −0 docs/source/api/holland.lib.mysql.holland.lib.rst
  188. +12 −0 docs/source/api/holland.lib.mysql.holland.rst
  189. +16 −0 docs/source/api/holland.rst
  190. +102 −0 docs/source/commands.rst
  191. +203 −0 docs/source/conf.py
  192. +139 −0 docs/source/config.rst
  193. +47 −0 docs/source/faq.rst
  194. +23 −0 docs/source/index.rst
  195. +35 −0 docs/source/intro.rst
  196. +76 −0 docs/source/overview.rst
  197. +31 −0 docs/source/provider_configs/compression.rst
  198. +39 −0 docs/source/provider_configs/databasefiltering.rst
  199. +6 −0 docs/source/provider_configs/example.rst
  200. +127 −0 docs/source/provider_configs/maatkit.rst
  201. +55 −0 docs/source/provider_configs/mysql-lvm.rst
  202. +25 −0 docs/source/provider_configs/mysqlconfig.rst
  203. +131 −0 docs/source/provider_configs/mysqldump.rst
  204. +63 −0 docs/source/provider_configs/mysqlhotcopy.rst
  205. +210 −0 docs/source/static/default.css
  206. +61 −0 docs/test_cases.txt
  207. +5 −0 holland-core/ChangeLog
  208. +63 −0 holland-core/LICENSE
  209. +23 −0 holland-core/README
  210. +75 −0 holland-core/docs/Makefile
  211. +188 −0 holland-core/docs/source/conf.py
  212. +25 −0 holland-core/docs/source/index.rst
  213. +1 −0  holland-core/holland/__init__.py
  214. +1 −0  holland-core/holland/backup/README
  215. +6 −0 holland-core/holland/backup/__init__.py
  216. +1 −0  holland-core/holland/commands/README
  217. +1 −0  holland-core/holland/commands/__init__.py
  218. +113 −0 holland-core/holland/commands/backup.py
  219. +52 −0 holland-core/holland/commands/help.py
  220. +62 −0 holland-core/holland/commands/list_backups.py
  221. +51 −0 holland-core/holland/commands/list_plugins.py
  222. +260 −0 holland-core/holland/commands/mk_config.py
  223. +94 −0 holland-core/holland/commands/purge.py
  224. +51 −0 holland-core/holland/commands/restore.py
  225. +1 −0  holland-core/holland/core/__init__.py
  226. +15 −0 holland-core/holland/core/backports/__init__.py
  227. +1,327 −0 holland-core/holland/core/backports/logging/__init__.py
  228. +310 −0 holland-core/holland/core/backports/logging/config.py
  229. +980 −0 holland-core/holland/core/backports/logging/handlers.py
  230. +1,689 −0 holland-core/holland/core/backports/optparse.py
  231. +1,325 −0 holland-core/holland/core/backports/subprocess.py
  232. +1,385 −0 holland-core/holland/core/backports/zipfile.py
  233. +188 −0 holland-core/holland/core/backup.py
  234. +64 −0 holland-core/holland/core/cmdshell.py
  235. +41 −0 holland-core/holland/core/command/__init__.py
  236. +220 −0 holland-core/holland/core/command/command.py
  237. +9 −0 holland-core/holland/core/config/__init__.py
  238. +68 −0 holland-core/holland/core/config/checks.py
  239. +178 −0 holland-core/holland/core/config/config.py
  240. +2,453 −0 holland-core/holland/core/config/configobj.py
  241. +1,462 −0 holland-core/holland/core/config/validate.py
  242. +19 −0 holland-core/holland/core/exceptions.py
  243. +42 −0 holland-core/holland/core/log.py
  244. +104 −0 holland-core/holland/core/plugin.py
  245. +298 −0 holland-core/holland/core/spool.py
  246. 0  holland-core/holland/core/util/__init__.py
  247. +82 −0 holland-core/holland/core/util/bootstrap.py
  248. +57 −0 holland-core/holland/core/util/fmt.py
  249. +136 −0 holland-core/holland/core/util/path.py
  250. +128 −0 holland-core/holland/core/util/template.py
  251. +460 −0 holland-core/holland/legacy/compression.py
  252. +123 −0 holland-core/holland/legacy/compression_util.py
  253. +202 −0 holland-core/holland/legacy/config.py
  254. +21 −0 holland-core/holland/legacy/log.py
  255. +99 −0 holland-core/holland/legacy/log_init.py
  256. +1 −0  holland-core/holland/lib/README
  257. +6 −0 holland-core/holland/lib/__init__.py
  258. +7 −0 holland-core/setup.cfg
  259. +41 −0 holland-core/setup.py
  260. +3 −0  holland-core/tests/__init__.py
  261. +201 −0 holland-core/tests/deprecrated_test_sysutils_helper.py
  262. +53 −0 holland-core/tests/test_bootstrap.py
  263. +46 −0 holland-core/tests/test_config.py
  264. +7 −0 plugins/ACTIVE
  265. +343 −0 plugins/holland.backup.example/LICENSE
  266. +6 −0 plugins/holland.backup.example/holland/__init__.py
  267. +6 −0 plugins/holland.backup.example/holland/backup/__init__.py
  268. +30 −0 plugins/holland.backup.example/holland/backup/example.py
  269. +7 −0 plugins/holland.backup.example/setup.cfg
  270. +29 −0 plugins/holland.backup.example/setup.py
  271. 0  plugins/holland.backup.example/tests/__init__.py
  272. +343 −0 plugins/holland.backup.maatkit/LICENSE
  273. +1 −0  plugins/holland.backup.maatkit/holland/__init__.py
  274. +1 −0  plugins/holland.backup.maatkit/holland/backup/__init__.py
  275. +289 −0 plugins/holland.backup.maatkit/holland/backup/maatkit.py
  276. +7 −0 plugins/holland.backup.maatkit/setup.cfg
  277. +28 −0 plugins/holland.backup.maatkit/setup.py
Sorry, we could not display the entire diff because too many files (501) changed.
View
3  .gitignore
@@ -0,0 +1,3 @@
+*pyc
+*egg*
+build/
View
235 ChangeLog
@@ -0,0 +1,235 @@
+2009-12-04 BJ Dierkes <wdierkes@rackspace.com>
+ * Added ./devtools/build_rpms.py to auto build rpms
+ from trunk.
+
+2009-12-01 Andrew Garner <andrew.garner@rackspace.com>
+ * Improved logging when plugins fail unexpectedly
+
+2009-11-24 Andrew Garner <andrew.garner@rackspace.com>
+ * holland backup --dry-run now checks available disk
+ space for whatever device /var/spool/hollland is on
+ compared to whatever the backup plugin reports
+ in plugin.estimate_backup_size()
+
+2009-11-13 Andrew Garner <andrew.garner@rackspace.com>
+ * holland log format is no longer hardcoded
+ * Use --base-dir rather than --basedir on new versions
+ of maatkit [#1554]
+
+2009-11-12 Andrew Garner <andrew.garner@rackspace.com>
+ * More cleanups for mysqldump restore plugin
+ * Add RHEL4 fixes for LVM plugin
+
+2009-11-08 Andrew Garner <andrew.garner@rackspace.com>
+ * LVM Backup plugin added [#1372]
+ * holland restore command added
+ * implemented mysqldump restore plugin
+ * implemented lvm restore plugin
+
+2009-11-08 Jason Kölker <jkoelker@rackspace.com>
+ * holland purge now asks for permission unless
+ the --force flag is used [#1541]
+ * Added Google logo to tomsay [#1543]
+
+2009-11-07 Andrew Garner <andrew.garner@rackspace.com>
+ * The mysqldump backup plugin now logs the database
+ and table filter patterns before starting backups
+ [#1539]
+ * holland list-plugins now lists holland.backup and
+ holland.commands plugins rather than just the
+ distributions.
+ * Raised the idle timeout in the mysqldump plugin from
+ 8 hours to 24 hours [#1537].
+
+2009-10-26 Andrew Garner <andrew.garner@rackspace.com>
+ * Cleaned up output and several bugfixes in mk-config
+ command [#1510]
+
+2009-10-26 Andrew Garner <andrew.garner@rackspace.com>
+ * Fixed table include/exclude patterns to support
+ both table and database.table matching [#1502]
+
+2009-10-09 Andrew Garner <andrew.garner@rackspace.com>
+ * Quoted values in my.cnf files are now parsed properly
+ [#1473]
+
+2009-09-15 Jason Kölker <jkoelker@rackspace.com>
+ * Improved commvault argument handling [#1466,#1471]
+ * Allow commvault to provide explicit backupsets
+
+2009-09-14 BJ Dierkes <wdierkes@rackspace.com>
+ * Holland 0.9.4 released.
+
+2009-09-11 Andrew Garner <andrew.garner@rackspace.com>
+ * Updated maatkit backup plugin to support option name
+ changes in mk-parallel-dump. Resolves [#1390]
+
+2009-08-26 Andrew Garner <andrew.garner@rackspace.com>
+ * Improved error message for config file syntax errors.
+ Resolves [#1443]
+ * Now Holland's backup module always logs the final
+ size estimated after applying the estimated-size-factor
+ adjustment. Resolves [#1399]
+ * In the mysqldump backup plugin, if bin-log-position is
+ requested on a slave, but log-bin is not enabled a warning
+ is logged, but this is no longer treated as an error.
+ Resolves [#1394]
+
+2009-08-20 Andrew Garner <andrew.garner@rackspace.com>
+ * Improved error message when STOP SLAVE fails, but
+ stop-slave was requested in the mysqldump backup
+ plugin. Resolves [#1435]
+
+2009-07-28 Andrew Garner <andrew.garner@rackspace.com>
+ * Improved commvault option parsing.
+
+2009-07-24 Andrew Garner <andrew.garner@rackspace.com>
+ * stderr output from compression pipes is now captured
+ and logged. Resolves [#1395]
+
+2009-07-23 Andrew Garner <andrew.garner@rackspace.com>
+ * When running the mysqldump backup plugin in dry-run
+ mode the slave is no longer stopped. Resolves [#1396]
+ * MySQL information_schema based database/table filtering
+ will filter databases more efficiently now.
+ Resolves [#1397]
+ * Using binlog-pos on a slave no longer requires binary
+ logging to be enabled.
+
+2009-07-13 BJ Dierkes <wdierkes@rackspace.com>
+ * Holland 0.9.3 released.
+
+
+2009-07-08 Andrew Garner <andrew.garner@rackspace.com>
+ * Fixed a couple bugs in holland.commands.mysql where
+ mysql-extract would fail. Resolves [#1379]
+ * Made estimated backup size adjustments more
+ consistent. [holland:backup] now accepts
+ a 'estimated-size-factor' that is a decimal
+ multiplier (default 1.0). This will apply
+ to any provider without any special support.
+ This also removes free-space-factor from the
+ mysqldump and mysqlhotcopy plugins and their
+ example configs. Resolves [#1378]
+ * Added 'backups-to-keep=1' to holland mk-config
+ generated configs. Resolves [#1374]
+ * Strip unknown opts when generating a my.cnf with
+ make_mycnf in holland.lib.mysql. Resolves [#1377]
+ * Fix defaults-extra-file typo in Maatkit plugin.
+ Resolves [#1377]
+ * Fix holland_cvmysqlsv to return correct exit code.
+ Resolves [#1375]
+ * Fix dry-run when using file-per-database option
+ Resolves [#1373]
+
+2009-06-11 BJ Dierkes <wdierkes@rackspace.com>
+ * Cleaned up --help output a bit. Resolves [#1333]
+ * Holland 0.9.2 released.
+
+2009-06-03 BJ Dierkes <wdierkes@rackspace.com>
+ * Holland 0.9.1 released.
+
+2009-05-06 Andrew Garner <andrew.garner@rackspace.com>
+ * Added holland.lib.mysql.client.connect method. This
+ rewrites my.cnf parameters to what MySQLdb.connect
+ expects. This allows passing option files from
+ holland.lib.mysql.option to the client.connect method
+ and connecting without necessarily flushing the options
+ to disk and using read_default_file=<path>.
+
+2009-04-19 Andrew Garner <andrew.garner@rackspace.com>
+ * Holland 0.3 released.
+
+2009-04-19 Andrew Garner <andrew.garner@rackspace.com>
+ * Converted holland.lib.mysql.client testcases to use
+ mock objects instead of direct dbapi connections
+ * Added Ian Bicking's multidict implementation to
+ holland.lib.common (used by holland.lib.mysql.option
+ but probably generally useful)
+ * mk-config now adds a [holland:backup] section to
+ generated configs
+
+2009-04-17 Andrew Garner <andrew.garner@rackspace.com>
+ * The mysqldump plugin now logs SHOW SLAVE STATUS
+ replication information to [mysql:replication] in
+ the backup.conf rather than to a separate replication.conf
+ file.
+ * Added "holland purge" command
+ * The holland.lib.mysql plugin distribution no longer lists
+ a setuptools dependency on MySQL-Python. This was breaking
+ on OS X in some cases and overall slowing down test runs.
+ * Fixed a compatibility problem in validate.py that was preventing
+ holland from running on python-2.3 (RHEL4)
+ * Further improvements to [.]my.cnf option-file processing in
+ holland.lib.mysql
+ * mysqlhotcopy and maatkit plugins now use the new my.cnf interface
+ in holland.lib.mysql.option
+ * /etc/holland/holland.conf is the default config file location
+ if an explicit --config is not specified or HOLLAND_CONFIG is
+ not set.
+ * [COMPATIBILITY] backupsets are now required to specify a
+ [holland:backup] section with a 'plugin' parameter to specify
+ the plugin to use (mysqldump,mysqlhotcopy,maatkit,...) rather
+ than a "root-level" provider option. This allows holland's config
+ files to be processed by non-ConfigObj standard ini-tools.
+ Further, [holland:backup] provides a bucket to record other
+ backup information about the backupset in the saved backup.conf
+ in the spool directory.
+ * Fixed a problem in "holland mk-config" that didn't generate optional
+ default=None parameters correctly.
+ * Fixed a minor rendering bug in Commmand.py's help formatting.
+ * Fixed a bug in holland help when referencing a command that does not
+ exist.
+ * Lots of bugfixes
+ * Coalesced holland.lib.archive and holland.lib.compression into
+ holland.lib.common distribution, along with a few other modules
+ that were previously in holland-core (but unused by holland-core)
+
+2009-04-16 Andrew Garner <andrew.garner@rackspace.com>
+ * Upgraded to ConfigObj 4.6.0, validate.py 1.0.0
+ * Cleaned up holland-core structure
+ * Added several in-progress plugins
+ * Moved many of the development "helper" scripts in trunk/
+ to trunk/dev-util/
+
+2009-04-01 Andrew Garner <andrew.garner@rackspace.com>
+ * Fixed maatkit plugin logging to capture stderr output
+ when running mk-parallel-dump
+
+2009-03-27 Andrew Garner <andrew.garner@rackspace.com>
+ * Added "holland mk-config" command to generate a backupset
+ config file.
+ * The mysqldump plugin now saves replication information from
+ SHOW SLAVE STATUS to replication.conf when backing up a slave
+ if bin-log-position is also defined.
+
+2009-03-24 Andrew Garner <andrew.garner@rackspace.com>
+ * Updated [.]my.cnf option-file processing in holland.lib.mysql
+ * Improved the performance of mysqlhotcopy backup (Issue #1191)
+ * Made stopping/starting slaves with mysqldump plugin more robust
+ * Added "defaults-extra-file" parameter to mysqldump plugin
+ This takes a list of my.cnf files which are used with the
+ backupset's mysqlclient config to produce a my.cnf auth-file.
+
+2009-03-11 Andrew Garner <andrew.garner@rackspace.com>
+ * Added "holland list-backups" command to show available backups
+ * Fixed backup purging functionality (Issue #1190)
+
+2009-03-10 Andrew Garner <andrew.garner@rackspace.com>
+
+ * Holland 0.2 release
+ * Bugfixes for Holland-Core
+ * Renamed holland.providers to holland.backup
+ * Refactored MySQLDump Backup Plugin; Now supports
+ advanced database and table filtering and many
+ new features previously broken in the config file
+ are now enabled
+ * Better logging throughout framework and plugins
+ * Enable dry-run support; Supported in all backup plugins
+ * Added MySQLHotcopy backup plugin
+ * Added Archive library plugin (used by MySQLHotcopy)
+ * Added Maatkit backup plugin (uses mk-parallel-dump)
+
+2009-02-20 Andrew Garner <andrew.garner@rackspace.com>
+
+ * Holland 0.1 released
View
57 INSTALL
@@ -0,0 +1,57 @@
+FIXME: This is rough and to the point.... needs fluff.
+
+
+UNPACK THE SOURCE:
+
+ you@linuxbox ~]$ tar -zxvf holland-x.y.z.tar.gz
+
+
+LOCAL DEVELOPMENT INSTALLATION:
+
+ you@linuxbox ~] cd ./holland-x.y.z
+ you@linuxbox holland-x.y.z] ./devtools/mkvirtenv.sh
+
+ Optionally, for target testing... add the --no-deps option to force
+ holland to rely on system dependencies (such as MySQL-python).
+
+
+SOURCE INSTALLATION:
+
+ Install Holland Core:
+
+ you@linuxbox ~]$ cd ./holland-x.y.z/holland-core
+ you@linuxbox holland-core]$ sudo python setup.py install
+
+
+ Install Holland Core Plugins:
+
+ you@linuxbox holland-x.y.z]$ cd plugins/holland.lib.common
+ you@linuxbox holland.lib.common]$ sudo python setup.py install
+
+ you@linuxbox holland-x.y.z]$ cd plugins/holland.lib.mysql
+ you@linuxbox holland.lib.mysql]$ sudo python setup.py install
+
+
+ Install Optional Plugins (see plugins/ACTIVE):
+
+ you@linuxbox holland-x.y.z]$ cd plugins/<plugin_dir>
+ you@linuxbox holland.<plugin_dir>]$ sudo python setup.py install
+
+
+
+UBUNTU INSTALLATION:
+
+ you@linuxbox holland-x.y.z]$ cp -a distribution/ubuntu/ubuntu.jaunty debian
+ you@linuxbox holland-x.y.z]$ dpkg-buildpackage -us -uc
+
+
+
+REDHAT/FEDORA/CENTOS/ETC INSTALLATION:
+
+ root@linuxbox ~]# cp -a holland-x.y.z/distribution/holland.spec /usr/src/redhat/SPECS
+ root@linuxbox ~]# cp -a holland-x.y.z.tar.gz /usr/src/redhat/SOURCES
+ root@linuxbox ~]# cd /usr/src/redhat
+ root@linuxbox /usr/src/redhat]# rpmbuild -ba SPECS/holland.spec
+
+
+
View
1  README
@@ -1 +0,0 @@
-Placeholder
View
0  addons/ACTIVE
No changes.
View
72 config/README
@@ -0,0 +1,72 @@
+Here are three approaches to a config file format:
+- Hierarchical
+Sections are organized in a single file in a hierarchy
+A backup-set can have sub-sections that define the config
+for various plugins:
+ ...
+ [backupsets]
+ [[default_backup]]
+ key = value
+ ...
+ [[[mysql-plugin]]]
+ host = remote-host
+ ...
+
+- Directory
+Hierarchy is broken down into different files in
+different directories:
+holland.conf
+# backupset configs
+backupsets/
+ default_backup.conf
+# global backup-plugin configs
+providers/
+ mysqldump.conf
+ lvm.conf
+ pgdump.conf
+ ...
+# global helper plugin library configs
+helpers/
+ mysqlclient.conf
+ lvmlib.conf
+ ...
+
+- Namespaces
+Single file, with standard ini sections named in a special
+way to denote namespaces.
+holland.conf:
+[holland]
+backup_directory = /<somepath>/
+
+[holland:logging]
+filename = /var/log/holland.log
+...
+
+[backup:default_backup]
+provider = mysqldump
+<mysqldump settings for 'default_backup'>
+[backup:default_backup:mysqlclient]
+# mysqldump backups utilize the mysqlclient lib
+host = 192.168.100.15 # some remote host
+user = remote-backup-user
+password = ...
+[backup:default_backup:compression]
+# suppose some general compression plugin we want to override
+cmd = lzop -U --fast
+
+[plugins:backup:mysqldump]
+# global options for the mysqldump backup plugin ("provider")
+# these are overriden in [backup:<name>]
+file_per_database = yes
+# exclude_databases = test
+# include_databases = production_db
+# regex = production_*
+# extra_args = --flush-privileges --ignore-table production_db.sessions
+
+[plugins:lib:mysqlclient]
+host = localhost
+socket = /var/lib/mysql/mysql.sock
+user = backup_user
+password = password
+ssl = no
+compress = no
View
35 config/backupsets/default.conf
@@ -0,0 +1,35 @@
+## Default Backup-Set
+##
+## Backs up all MySQL databases in a one-file-per-database fashion using
+## lightweight in-line compression and engine auto-detection. This backup-set
+## is designed to provide reliable backups "out of the box", however it is
+## generally advisable to create additional custom backup-sets to suit
+## one's specific needs.
+##
+## For more inforamtion about backup-sets, please consult the online Holland
+## documentation. Fully-commented example backup-sets are also provided, by
+## default, in /etc/holland/backupsets/examples.
+
+[holland:backup]
+plugin = mysqldump
+backups-to-keep = 1
+estimated-size-factor = 1.0
+
+# This section defines the configuration options specific to the backup
+# plugin. In other words, the name of this section should match the name
+# of the plugin defined above.
+[mysqldump]
+file-per-database = yes
+#lock-method = auto-detect
+#databases = "*"
+#tables = "*"
+#stop-slave = no
+#bin-log-position = no
+
+# The following section is for compression. The default, unless the
+# mysqldump provider has been modified, is to use inline fast gzip
+# compression (which is identical to the commented section below).
+#[compression]
+#method = gzip
+#inline = yes
+#level = 1
View
13 config/backupsets/examples/example.conf
@@ -0,0 +1,13 @@
+## Example Backup-Set
+##
+## This uses the 'example' provider. This provider is purely for use as a
+## template for creating additional providers and otherwise does nothing.
+
+[holland:backup]
+plugin = example
+
+# This section defines the configuration options specific to the backup
+# plugin. In other words, the name of this section should match the name
+# of the plugin defined above.
+[example]
+# No options
View
95 config/backupsets/examples/maatkit.conf
@@ -0,0 +1,95 @@
+## Holland Maatkit Example Backup-Set
+##
+## An example of how to use the Maatkti provider to perform parallel
+## backups. Note that many of these options apply to the 'mk-parallel-dump'
+## utility directly. As a result, for full elaboration on what each option
+## does, it is highly recommended to read the Maatkit documentation for
+## 'mk-parallel-dump'.
+##
+## Many of these options have global defaults which can be found in the
+## configuration file for the provider (which can be found, by default
+## in /etc/holland/providers).
+##
+## This is experimental, and largely recommended for MySQL slaves
+
+[holland:backup]
+plugin = maatkit
+backups-to-keep = 1
+
+# This section defines the configuration options specific to the backup
+# plugin. In other words, the name of this section should match the name
+# of the plugin defined above.
+[maatkit]
+
+## One of: flush-lock, lock-tables, single-transaction, auto-detect, none
+##
+## flush-lock will run a FLUSH TABLES WITH READ LOCK prior to the backup
+##
+## lock-tables will instruct 'mysqldump' to lock all tables involved
+## in the backup.
+##
+## None will completely disable locking. This is generally only viable
+## on a MySQL slave and only after traffic has been diverted, or slave
+## services suspended.
+#lock-method = flush-lock
+
+## Whether or not to backup the largest tables first.
+#biggestfirst = yes
+
+## Whether to record the binary log name and position at the time of the
+## backup.
+#binlogpos = yes
+
+## Which default character set to use to conenct to MySQL with.
+#charset = utf8
+
+## Specifies the number of rows or size that each backup file will be.
+## Specifying a G, M or k after the number will cause Holland to split the
+## chunks by size. Otherwise, it will split by rows.# Consult the Maatkit
+## documentation on 'mk-parallel-dump' for more information.
+#chunksize = 5000
+
+## Database and table filtering. Note that this differs from the standard
+## database filtering found in other providers! Consult the Maatkit
+## documentation on 'mk-parallel-dump' for more information.
+#databases =
+#ignoredb =
+#dbregex =
+#tables =
+#tblregex =
+#ignoretbl =
+
+## Maatkit also supports engine filtering.
+## FEDERATED,MRG_MyISAM are skipped by default
+#ignoreengine = FEDERATED,MRG_MyISAM
+
+## The number of threads to use in the parallel backup operation.
+#numthread = 2
+
+## Whether to stop the slave before commencing with the backup
+#stopslave = no
+
+## Whether or not to run FLUSH LOGS before a backup. Be careful when using
+## this option with setperdb as that can cause a number flushes to be
+## executed, which can hinder performance and create a large number of
+## binary logs.
+#flushlog = no
+
+## Whether or not to compress the backups using gzip. Currently gzip is the
+## only option because it is handled by 'mk-parallel-dump' and not by
+## Holland.
+#gzip = yes
+
+## Whether or not to backup each database into its own file.
+#setperdb = no
+
+## MySQL connection settings. Note that these can be inherited from the
+## provider itself allowing for global defaults. Providing connection
+## information for a backup-set can often be helpful when, for instance
+## a backup-set is backing up a remote MySQL server.
+#[mysql:client]
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
71 config/backupsets/examples/mysql-lvm.conf
@@ -0,0 +1,71 @@
+## MySQL LVM Backup Example Backup-Set
+##
+## This implements a standard MySQL + LVM Backup. For the most part,
+## the snapshot options need no modification. Changing the recovery
+## and locking options may be occasionally useful. The plugin is designed
+## to fit the majority of configurations without needing to make any
+## modifications, however.
+
+## Global Holland Settings
+[holland:backup]
+plugin = mysql-lvm
+backups-to-keep = 1
+estimated-size-factor = 1.0
+
+## LVM Backup Specific Settings
+[mysql-lvm]
+## The size of the snapshot itself. By default it is 20% of the size of
+## the MySQL LVM mount or the remaining free-space in the Volume Group
+## (if there is less than 20% available).
+## If snapshot-size is defined, the number represents the size of the
+## snapshot in megabytes
+#snapshot-size = ""
+
+## The name of the snapshot, the default being the name of the MySQL LVM
+## volume + "_snapshot" (ie Storage-MySQL_snapshot)
+#snapshot-name = "" # no default
+
+## Where to mount the snapshot. By default a randomly generated directory
+## under /tmp is used.
+#snapshot-mountpoint = "" # no default
+
+## Whether or not to run an InnoDB recovery operation. This avoids needing
+## to do so during a restore, though will make the backup process itself
+## take longer.
+#innodb-recovery = False
+
+## Whether or not to run a FLUSH TABLES WITH READ LOCK to grab various
+## bits of information (such as the binary log name and position). Disabling
+## this requires that binary logging is disabled and InnoDB is being used
+## exclusively. Otherwise, it is possible that the backup could contain
+## crashed tables.
+#lock-tables = True
+
+## Whether or not to run a FLUSH TABLES before running the full
+## FLUSH TABLES WITH READ LOCK. Should make the FLUSH TABLES WITH READ LOCK
+## operation a bit faster.
+#extra-flush-tables = True
+
+## Compression Settings
+[compression]
+
+## compress method: gzip, bzip2, lzop, or xz
+## Which compression method to use, which can be either gzip, bzip2, or lzop.
+## Note that lzop is not often installed by default on many Linux
+## distributions and may need to be installed separately.
+method = gzip
+
+## What compression level to use. Lower numbers mean faster compression,
+## though also generally a worse compression ratio. Generally, levels 1-3
+## are considered fairly fast and still offer good compression for textual
+## data. Levels above 7 can often cause a larger impact on the system due to
+## needing much more CPU resources. Setting the level to 0 effectively
+## disables compresion.
+level = 1
+
+#[mysql:client]
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
127 config/backupsets/examples/mysqldump.conf
@@ -0,0 +1,127 @@
+## Holland mysqldump Example Backup-Set
+##
+## This implements a vanilla backup-set using the mysqldump provider which,
+## in turn, uses the 'mysqldump' utility.
+##
+## Many of these options have global defaults which can be found in the
+## configuration file for the provider (which can be found, by default
+## in /etc/holland/providers).
+
+[holland:backup]
+plugin = mysqldump
+backups-to-keep = 1
+estimated-size-factor = 1.0
+
+# This section defines the configuration options specific to the backup
+# plugin. In other words, the name of this section should match the name
+# of the plugin defined above.
+[mysqldump]
+
+## Override the path where we can find mysql command line utilities
+#mysql-binpath = /usr/bin/mysqldump
+
+## One of: flush-lock, lock-tables, single-transaction, auto-detect, none
+##
+## flush-lock will run a FLUSH TABLES WITH READ LOCK prior to the backup
+##
+## lock-tables will instruct 'mysqldump' to lock all tables involved
+## in the backup.
+##
+## single-transaction will force running a backup within a transaction.
+## This allows backing up of transactional tables without imposing a lock
+## howerver will NOT properly backup non-transacitonal tables.
+##
+## Auto-detect will choose single-transaction unless Holland finds
+## non-transactional tables in the backup-set.
+##
+## None will completely disable locking. This is generally only viable
+## on a MySQL slave and only after traffic has been diverted, or slave
+## services suspended.
+lock-method = auto-detect
+
+## comma-delimited glob patterns for matching databases
+## only databases matching these patterns will be backed up
+## default: include everything
+databases = "*"
+
+## comma-delimited glob patterns to exclude particular
+## databases
+#exclude-databases =
+
+## only include the specified tables
+tables = "*"
+
+## exclude specific tables
+#exclude-tables = ""
+
+## Whether to dump routines explicitly
+## (routines are implicitly included in the mysql database)
+dump-routines = no
+
+## Whether to dump events explicitly.
+## Note that this feature requires MySQL 5.1 or later.
+dump-events = no
+
+## Whether to stop the slave before commencing with the backup
+stop-slave = no
+
+## Whether to record the binary log name and position at the time of the
+## backup.
+bin-log-position = no
+
+## Whether or not to run FLUSH LOGS in MySQL with the backup. When FLUSH
+## LOGS is actually executed depends on which if database filtering is being
+## used and whether or not file-per-database is enabled. Generally speaking,
+## it does not make sense to use flush-logs with file-per-database since the
+## binary logs will not be consistent with the backup.
+flush-logs = no
+
+## Whether to run a separate mysqldump for each database. Note that while
+## this may initially sound like a good idea, it is far simpler to backup
+## all databases in one file, although that makes the restore process
+## more difficult when only certain data needs to be restored.
+file-per-database = no
+
+## any additional options to the 'mysqldump' command-line utility
+## these should show up exactly as they are on the command line
+## e.g.: --flush-privileges --reset-master
+additional-options = ""
+
+## Compression Settings
+[compression]
+
+## compress method: gzip, bzip2, lzop, or xz
+## Which compression method to use, which can be either gzip, bzip2, or lzop.
+## Note that lzop is not often installed by default on many Linux
+## distributions and may need to be installed separately.
+method = gzip
+
+## Whether to compress data as it is provided from 'mysqldump', or to
+## compress after a dump has finished. In general, it is often better to use
+## inline compression. The overhead, particularly when using a lower
+## compression level, is often minial since the entire process is often I/O
+## bound (as opposed to being CPU bound).
+inline = yes
+
+## What compression level to use. Lower numbers mean faster compression,
+## though also generally a worse compression ratio. Generally, levels 1-3
+## are considered fairly fast and still offer good compression for textual
+## data. Levels above 7 can often cause a larger impact on the system due to
+## needing much more CPU resources. Setting the level to 0 effectively
+## disables compresion.
+level = 1
+
+## If the path to the compression program is in a non-standard location,
+## or not in the system-path, you can provide it here.
+#bin-path = /usr/bin/gzip
+
+## MySQL connection settings. Note that these can be inherited from the
+## provider itself allowing for global defaults. Providing connection
+## information for a backup-set can often be helpful when, for instance
+## a backup-set is backing up a remote MySQL server.
+#[mysql:client]
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
85 config/backupsets/examples/mysqlhotcopy.conf
@@ -0,0 +1,85 @@
+## Holland MySQL Hot Copy Example Backup-Set
+##
+## Uses the 'mysqlhotcopy' plugin which allows for copying of the raw data
+## of non-transactional tables.
+##
+## Many of these options have global defaults which can be found in the
+## configuration file for the provider (which can be found, by default
+## in /etc/holland/providers).
+
+[holland:backup]
+plugin = mysqlhotcopy
+backups-to-keep = 1
+estimated-size-factor = 1.0
+
+# This section defines the configuration options specific to the backup
+# plugin. In other words, the name of this section should match the name
+# of the plugin defined above.
+[mysqlhotcopy]
+
+## Override the path where we can find mysql command line utilities
+#mysql-binpath = /usr/bin/mysqldump
+
+## One of: flush-lock, lock-tables, none
+##
+## flush-lock will run a FLUSH TABLES WITH READ LOCK prior to the backup
+##
+## lock-tables will lock all tables involved in the backup.
+##
+## None will completely disable locking. This is generally only viable
+## on a MySQL slave and only after traffic has been diverted, or slave
+## services suspended.
+#lock-method = lock-tables
+
+## comma-delimited glob patterns for matching databases
+## only databases matching these patterns will be backed up
+## default: include everything
+databases = "*"
+
+## comma-delimited glob patterns to exclude particular
+## databases
+#exclude-databases =
+
+## only include the specified tables
+tables = "*"
+
+## exclude specific tables
+#exclude-tables = ""
+
+## Only backup first 2K of MYI files. Only the first 2K of an MYI file is
+## needed in order to restore from a backup. However, if this option is
+## enabled, the tables must be repaired using a REPAIR TABLE in MySQL
+## before they will be usable.
+#partial-indexes = false
+
+## Whether or not to compress or tar the resulting backups. Three options
+## are avaliable: (dir)ectory (zip)file or (tar)file
+##
+## dir does no compression and copies out the resulting table-defitions,
+## data and index files into a directory correspndong to the database
+## directory.
+##
+## zip will compress the backups into a ZIP archive. Since ZIPs support
+## random access, this can save space but still allow for restoring only
+## certain tables quickly.
+##
+## tar will tar up and compress the backups (using gzip).
+#archive-method = dir
+
+## Whether to stop the slave before commencing with the backup
+stop-slave = no
+
+## Whether to record the binary log name and position at the time of the
+## backup.
+bin-log-position = no
+
+## MySQL connection settings. Note that these can be inherited from the
+## provider itself allowing for global defaults. Providing connection
+## information for a backup-set can often be helpful when, for instance
+## a backup-set is backing up a remote MySQL server.
+#[mysql:client]
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
27 config/holland.conf
@@ -0,0 +1,27 @@
+## Root holland config file
+[holland]
+
+## Paths where holland plugins may be found.
+## Can be comma separated
+plugin_dirs = /usr/share/holland/plugins
+
+## Top level directory where backups are held
+backup_directory = /var/spool/holland
+
+## List of enabled backup sets. Can be comma separated.
+## Read from <config_dir>/backupsets/<name>.conf
+# backupsets = example, traditional, parallel_backups, non_transactional
+backupsets = default
+
+# Define a umask for file generated by holland
+umask = 0007
+
+# Define a path for holland and its spawned processes
+path = /usr/local/bin:/usr/local/sbin:/bin:/sbin:/usr/bin:/usr/sbin
+
+[logging]
+## where to write the log
+filename = /var/log/holland/holland.log
+
+## debug, info, warning, error, critical (case insensitive)
+level = info
View
5 config/providers/README
@@ -0,0 +1,5 @@
+The providers directory contains global
+configs for each backup provider. These
+settings are merged with those set in
+individual backupsets. The settings in
+backupsets take precedence.
View
3  config/providers/example.conf
@@ -0,0 +1,3 @@
+## Example Provider
+##
+## There are no settings available at this time.
View
77 config/providers/maatkit.conf
@@ -0,0 +1,77 @@
+## Global settings for the Maatkit provider
+##
+## Unless overwritten, all backup-sets implementing this provider will use
+## the following settings.
+
+[maatkit]
+
+## One of: flush-lock, lock-tables, single-transaction, auto-detect, none
+##
+## flush-lock will run a FLUSH TABLES WITH READ LOCK prior to the backup
+##
+## lock-tables will instruct 'mysqldump' to lock all tables involved
+## in the backup.
+##
+## None will completely disable locking. This is generally only viable
+## on a MySQL slave and only after traffic has been diverted, or slave
+## services suspended.
+#lock-method = flush-lock
+
+## Whether or not to backup the largest tables first.
+#biggestfirst = yes
+
+## Whether to record the binary log name and position at the time of the
+## backup.
+#binlogpos = yes
+
+## Which character set to use to conenct to MySQL with.
+#charset = utf8
+
+## Either the number of rows to backup per file; or the size of the files
+## if G, M, or k is specified after the number.
+## Consult the Maatkit documentation on 'mk-parallel-dump' for more
+## information.
+#chunksize = 5000
+
+## Database and table filtering. Note that this differs from the standard
+## database filtering found in other providers! Consult the Maatkit
+## documentation on 'mk-parallel-dump' for more information.
+#databases =
+#ignoredb =
+#dbregex =
+#tables =
+#tblregex =
+#ignoretbl =
+
+## Maatkit also supports engine filtering.
+## FEDERATED,MRG_MyISAM are skipped by default
+#ignoreengine = FEDERATED,MRG_MyISAM
+
+## The number of threads to use in the parallel backup operation.
+#numthread = 2
+
+## Whether to stop the slave before commencing with the backup
+#stopslave = no
+
+## Whether or not to run FLUSH LOGS before a backup. Be careful when using
+## this option with setperdb as that can cause a number flushes to be
+## executed, which can hinder performance and create a large number of
+## binary logs.
+#flushlog = no
+
+## Whether or not to compress the backups using gzip.
+#gzip = yes
+
+## Whether or not to backup each database into its own file.
+#setperdb = no
+
+## MySQL connection settings. Note that Holland will try ot read from
+## the provided files defined in the 'defaults-extra-file', although
+## explicitly defining the connection information here will take precedence.
+[mysql:client]
+defaults-extra-file = ~/.my.cnf,
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
49 config/providers/mysql-lvm.conf
@@ -0,0 +1,49 @@
+## MySQL LVM Backup Provider
+##
+## All settings have generated defaults so nothing need be configured here
+## Instead, it is best to make changes in a specific backup-set.
+
+[mysql-lvm]
+## The size of the snapshot itself. By default it is 20% of the size of
+## the MySQL LVM mount or the remaining free-space in the Volume Group
+## (if there is less than 20% available).
+## If snapshot-size is defined, the number represents the size of the
+## snapshot in megabytes.
+snapshot-size = ""
+
+## The name of the snapshot, the default being the name of the MySQL LVM
+## volume + "_snapshot" (ie Storage-MySQL_snapshot)
+snapshot-name = "" # no default
+
+## Where to mount the snapshot. By default a randomly generated directory
+## under /tmp is used.
+snapshot-mountpoint = "" # no default
+
+## Whether or not to run an InnoDB recovery operation. This avoids needing
+## to do so during a restore, though will make the backup process itself
+## take longer.
+innodb-recovery = False
+
+## Whether or not to run a FLUSH TABLES WITH READ LOCK to grab various
+## bits of information (such as the binary log name and position). Disabling
+## this requires that binary logging is disabled and InnoDB is being used
+## exclusively. Otherwise, it is possible that the backup could contain
+## crashed tables.
+lock-tables = True
+
+## Whether or not to run a FLUSH TABLES before running the full
+## FLUSH TABLES WITH READ LOCK. Should make the FLUSH TABLES WITH READ LOCK
+## operation a bit faster.
+extra-flush-tables = True
+
+[compression]
+method = gzip
+level = 1
+
+[mysql:client]
+defaults-file = /root/.my.cnf
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
124 config/providers/mysqldump.conf
@@ -0,0 +1,124 @@
+## Global settings for the mysqldump provider
+##
+## Unless overwritten, all backup-sets implementing this provider will use
+## the following settings.
+
+[mysqldump]
+
+## Override the path where we can find mysql command line utilities
+#mysql-binpath = /usr/bin/mysqldump
+
+## One of: flush-lock, lock-tables, single-transaction, auto-detect, none
+##
+## flush-lock will place a global lock on all tables involved in the backup
+## regardless of whether or not they are in the backup-set. If
+## file-per-database is enabled, then flush-lock will lock all tables
+## for every database being backed up. In other words, this option may not
+## make much sense when using file-per-database.
+##
+## lock-tables will lock all tables involved in the backup. If
+## file-per-database is enabled, then lock-tables will only lock all the
+## tables associated with that database.
+##
+## single-transaction will force running a backup within a transaction.
+## This allows backing up of transactional tables without imposing a lock
+## howerver will NOT properly backup non-transacitonal tables.
+##
+## Auto-detect will choose single-transaction unless Holland finds
+## non-transactional tables in the backup-set.
+##
+## None will completely disable locking. This is generally only viable
+## on a MySQL slave and only after traffic has been diverted, or slave
+## services suspended.
+lock-method = auto-detect
+
+## comma-delimited glob patterns for matching databases
+## only databases matching these patterns will be backed up
+## default: include everything
+#databases = "*"
+
+## comma-delimited glob patterns to exclude particular
+## databases
+#exclude-databases =
+
+## only include the specified tables
+#tables = "*"
+
+## exclude specific tables
+#exclude-tables = ""
+
+## Whether to dump routines explicitly
+## (routines are implicitly included in the mysql database)
+dump-routines = no
+
+## Whether to dump events explicitly.
+## Note that this feature requires MySQL 5.1 or later.
+dump-events = no
+
+## Whether to stop the slave before commencing with the backup
+stop-slave = no
+
+## Whether to record the binary log name and position at the time of the
+## backup.
+bin-log-position = no
+
+## Whether or not to run FLUSH LOGS in MySQL with the backup. When FLUSH
+## LOGS is actually executed depends on which if database filtering is being
+## used and whether or not file-per-database is enabled. Generally speaking,
+## it does not make sense to use flush-logs with file-per-database since the
+## binary logs will not be consistent with the backup.
+flush-logs = no
+
+## Whether to run a separate mysqldump for each database. Note that while
+## this may initially sound like a good idea, it is far simpler to backup
+## all databases in one file, although that makes the restore process
+## more difficult when only certain data needs to be restored.
+file-per-database = no
+
+## any additional options to the 'mysqldump' command-line utility
+## these should show up exactly as they are on the command line
+## e.g.: --flush-privileges --reset-master
+additional-options = ""
+
+## Compression Settings
+[compression]
+
+## compress method: gzip, bzip2 or lzop
+## Which compression method to use, which can be either gzip, bzip2, or lzop.
+## Note that lzop is not often installed by default on many Linux
+## distributions and may need to be installed separately.
+method = gzip
+
+## Whether to compress data as it is provided from 'mysqldump', or to
+## compress after a dump has finished. In general, it is often better to use
+## inline compression. The overhead, particularly when using a lower
+## compression level, is often minial since the entire process is often I/O
+## bound (as opposed to being CPU bound).
+inline = yes
+
+## What compression level to use. Lower numbers mean faster compression,
+## though also generally a worse compression ratio. Generally, levels 1-3
+## are considered fairly fast and still offer good compression for textual
+## data. Levels above 7 can often cause a larger impact on the system due to
+## needing much more CPU resources. Setting the level to 0 effectively
+## disables compresion.
+level = 1
+
+## If the path to the compression program is in a non-standard location,
+## or not in the system-path, you can provide it here.
+##
+## FIXME: Currently not implemented, compression binary is looked up by
+## which.
+##
+#bin-path = /usr/bin/gzip
+
+## MySQL connection settings. Note that Holland will try ot read from
+## the provided files defined in the 'defaults-extra-file', although
+## explicitly defining the connection inforamtion here will take precedence.
+[mysql:client]
+defaults-extra-file = /root/.my.cnf,~/.my.cnf,
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
73 config/providers/mysqlhotcopy.conf
@@ -0,0 +1,73 @@
+## Global settings for the maatkit provider
+##
+## Unless overwritten, all backup-sets implementing this provider will use
+## the following settings.
+
+[mysqlhotcopy]
+
+## Override the path where we can find mysql command line utilities
+#mysql-binpath = /usr/bin/mysqldump
+
+## One of: flush-lock, lock-tables, none
+##
+## flush-lock will run a FLUSH TABLES WITH READ LOCK prior to the backup
+##
+## lock-tables will lock all tables involved in the backup.
+##
+## None will completely disable locking. This is generally only viable
+## on a MySQL slave and only after traffic has been diverted, or slave
+## services suspended.
+#lock-method = lock-tables
+
+## comma-delimited glob patterns for matching databases
+## only databases matching these patterns will be backed up
+## default: include everything
+databases = "*"
+
+## comma-delimited glob patterns to exclude particular
+## databases
+#exclude-databases =
+
+## only include the specified tables
+tables = "*"
+
+## exclude specific tables
+#exclude-tables = ""
+
+## Only backup first 2K of MYI files. Only the first 2K of an MYI file is
+## needed in order to restore from a backup. However, if this option is
+## enabled, the tables must be repaired using a REPAIR TABLE in MySQL
+## before they will be usable.
+#partial-indexes = false
+
+## Whether or not to compress or tar the resulting backups. Three options
+## are avaliable: (dir)ectory (zip)file or (tar)file
+##
+## dir does no compression and copies out the resulting table-defitions,
+## data and index files into a directory correspndong to the database
+## directory.
+##
+## zip will compress the backups into a ZIP archive. Since ZIPs support
+## random access, this can save space but still allow for restoring only
+## certain tables quickly.
+##
+## tar will tar up and compress the backups (using gzip).
+#archive-method = dir
+
+## Whether to stop the slave before commencing with the backup
+stop-slave = no
+
+## Whether to record the binary log name and position at the time of the
+## backup.
+bin-log-position = no
+
+## MySQL connection settings. Note that Holland will try ot read from
+## the provided files defined in the 'defaults-extra-file', although
+## explicitly defining the connection inforamtion here will take precedence.
+[mysql:client]
+defaults-extra-file = ~/.my.cnf,
+#user = hollandbackup
+#password = "hollandpw"
+#socket = /tmp/mysqld.sock
+#host = localhost
+#port = 3306
View
29 devtools/README
@@ -0,0 +1,29 @@
+This directory contains various utility scripts
+used in Holland development.
+
+quick_virtinstall.sh
+====================
+Setups up a virtual python environment using
+virtualenv and installs holland-core and all
+holland plugins listed in plugins/ACTIVE into
+the new virtual environment, along with a few
+dependencies (MySQL_python, etc.)
+
+test_it.sh
+====================
+Runs through the various packages (holland-core,
+plugins/*) and runs python setup.py test.
+
+load_it.py
+====================
+Simple script to load the holland environment
+and associated plugin directory. Useful for
+check if modules are importing properly:
+python -i load_it.py
+>>> import holland.backup.mysqldump
+>>>
+
+build_holland_doc.sh
+====================
+Quick script to be run out of cron to build
+documentation of every tag, branch, and trunk.
View
1,482 devtools/_virtualenv.py
@@ -0,0 +1,1482 @@
+#!/usr/bin/env python
+"""Create a "virtual" Python installation
+"""
+
+virtualenv_version = "1.4.3"
+
+import sys
+import os
+import optparse
+import re
+import shutil
+import logging
+import distutils.sysconfig
+try:
+ import subprocess
+except ImportError, e:
+ if sys.version_info <= (2, 3):
+ print 'ERROR: %s' % e
+ print 'ERROR: this script requires Python 2.4 or greater; or at least the subprocess module.'
+ print 'If you copy subprocess.py from a newer version of Python this script will probably work'
+ sys.exit(101)
+ else:
+ raise
+try:
+ set
+except NameError:
+ from sets import Set as set
+
+join = os.path.join
+py_version = 'python%s.%s' % (sys.version_info[0], sys.version_info[1])
+is_jython = sys.platform.startswith('java')
+expected_exe = is_jython and 'jython' or 'python'
+
+REQUIRED_MODULES = ['os', 'posix', 'posixpath', 'ntpath', 'genericpath',
+ 'fnmatch', 'locale', 'encodings', 'codecs',
+ 'stat', 'UserDict', 'readline', 'copy_reg', 'types',
+ 're', 'sre', 'sre_parse', 'sre_constants', 'sre_compile',
+ 'lib-dynload', 'config', 'zlib']
+
+if sys.version_info[:2] == (2, 6):
+ REQUIRED_MODULES.extend(['warnings', 'linecache', '_abcoll', 'abc'])
+if sys.version_info[:2] <= (2, 3):
+ REQUIRED_MODULES.extend(['sets', '__future__'])
+
+class Logger(object):
+
+ """
+ Logging object for use in command-line script. Allows ranges of
+ levels, to avoid some redundancy of displayed information.
+ """
+
+ DEBUG = logging.DEBUG
+ INFO = logging.INFO
+ NOTIFY = (logging.INFO+logging.WARN)/2
+ WARN = WARNING = logging.WARN
+ ERROR = logging.ERROR
+ FATAL = logging.FATAL
+
+ LEVELS = [DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL]
+
+ def __init__(self, consumers):
+ self.consumers = consumers
+ self.indent = 0
+ self.in_progress = None
+ self.in_progress_hanging = False
+
+ def debug(self, msg, *args, **kw):
+ self.log(self.DEBUG, msg, *args, **kw)
+ def info(self, msg, *args, **kw):
+ self.log(self.INFO, msg, *args, **kw)
+ def notify(self, msg, *args, **kw):
+ self.log(self.NOTIFY, msg, *args, **kw)
+ def warn(self, msg, *args, **kw):
+ self.log(self.WARN, msg, *args, **kw)
+ def error(self, msg, *args, **kw):
+ self.log(self.WARN, msg, *args, **kw)
+ def fatal(self, msg, *args, **kw):
+ self.log(self.FATAL, msg, *args, **kw)
+ def log(self, level, msg, *args, **kw):
+ if args:
+ if kw:
+ raise TypeError(
+ "You may give positional or keyword arguments, not both")
+ args = args or kw
+ rendered = None
+ for consumer_level, consumer in self.consumers:
+ if self.level_matches(level, consumer_level):
+ if (self.in_progress_hanging
+ and consumer in (sys.stdout, sys.stderr)):
+ self.in_progress_hanging = False
+ sys.stdout.write('\n')
+ sys.stdout.flush()
+ if rendered is None:
+ if args:
+ rendered = msg % args
+ else:
+ rendered = msg
+ rendered = ' '*self.indent + rendered
+ if hasattr(consumer, 'write'):
+ consumer.write(rendered+'\n')
+ else:
+ consumer(rendered)
+
+ def start_progress(self, msg):
+ assert not self.in_progress, (
+ "Tried to start_progress(%r) while in_progress %r"
+ % (msg, self.in_progress))
+ if self.level_matches(self.NOTIFY, self._stdout_level()):
+ sys.stdout.write(msg)
+ sys.stdout.flush()
+ self.in_progress_hanging = True
+ else:
+ self.in_progress_hanging = False
+ self.in_progress = msg
+
+ def end_progress(self, msg='done.'):
+ assert self.in_progress, (
+ "Tried to end_progress without start_progress")
+ if self.stdout_level_matches(self.NOTIFY):
+ if not self.in_progress_hanging:
+ # Some message has been printed out since start_progress
+ sys.stdout.write('...' + self.in_progress + msg + '\n')
+ sys.stdout.flush()
+ else:
+ sys.stdout.write(msg + '\n')
+ sys.stdout.flush()
+ self.in_progress = None
+ self.in_progress_hanging = False
+
+ def show_progress(self):
+ """If we are in a progress scope, and no log messages have been
+ shown, write out another '.'"""
+ if self.in_progress_hanging:
+ sys.stdout.write('.')
+ sys.stdout.flush()
+
+ def stdout_level_matches(self, level):
+ """Returns true if a message at this level will go to stdout"""
+ return self.level_matches(level, self._stdout_level())
+
+ def _stdout_level(self):
+ """Returns the level that stdout runs at"""
+ for level, consumer in self.consumers:
+ if consumer is sys.stdout:
+ return level
+ return self.FATAL
+
+ def level_matches(self, level, consumer_level):
+ """
+ >>> l = Logger()
+ >>> l.level_matches(3, 4)
+ False
+ >>> l.level_matches(3, 2)
+ True
+ >>> l.level_matches(slice(None, 3), 3)
+ False
+ >>> l.level_matches(slice(None, 3), 2)
+ True
+ >>> l.level_matches(slice(1, 3), 1)
+ True
+ >>> l.level_matches(slice(2, 3), 1)
+ False
+ """
+ if isinstance(level, slice):
+ start, stop = level.start, level.stop
+ if start is not None and start > consumer_level:
+ return False
+ if stop is not None or stop <= consumer_level:
+ return False
+ return True
+ else:
+ return level >= consumer_level
+
+ #@classmethod
+ def level_for_integer(cls, level):
+ levels = cls.LEVELS
+ if level < 0:
+ return levels[0]
+ if level >= len(levels):
+ return levels[-1]
+ return levels[level]
+
+ level_for_integer = classmethod(level_for_integer)
+
+
+logger = Logger([(Logger.level_for_integer(2), sys.stdout)])
+
+def mkdir(path):
+ if not os.path.exists(path):
+ logger.info('Creating %s', path)
+ os.makedirs(path)
+ else:
+ logger.info('Directory %s already exists', path)
+
+def copyfile(src, dest, symlink=True):
+ if not os.path.exists(src):
+ # Some bad symlink in the src
+ logger.warn('Cannot find file %s (bad symlink)', src)
+ return
+ if os.path.exists(dest):
+ logger.debug('File %s already exists', dest)
+ return
+ if not os.path.exists(os.path.dirname(dest)):
+ logger.info('Creating parent directories for %s' % os.path.dirname(dest))
+ os.makedirs(os.path.dirname(dest))
+ if symlink and hasattr(os, 'symlink'):
+ logger.info('Symlinking %s', dest)
+ os.symlink(os.path.abspath(src), dest)
+ else:
+ logger.info('Copying to %s', dest)
+ if os.path.isdir(src):
+ shutil.copytree(src, dest, True)
+ else:
+ shutil.copy2(src, dest)
+
+def writefile(dest, content, overwrite=True):
+ if not os.path.exists(dest):
+ logger.info('Writing %s', dest)
+ f = open(dest, 'wb')
+ f.write(content)
+ f.close()
+ return
+ else:
+ f = open(dest, 'rb')
+ c = f.read()
+ f.close()
+ if c != content:
+ if not overwrite:
+ logger.notify('File %s exists with different content; not overwriting', dest)
+ return
+ logger.notify('Overwriting %s with new content', dest)
+ f = open(dest, 'wb')
+ f.write(content)
+ f.close()
+ else:
+ logger.info('Content %s already in place', dest)
+
+def rmtree(dir):
+ if os.path.exists(dir):
+ logger.notify('Deleting tree %s', dir)
+ shutil.rmtree(dir)
+ else:
+ logger.info('Do not need to delete %s; already gone', dir)
+
+def make_exe(fn):
+ if hasattr(os, 'chmod'):
+ oldmode = os.stat(fn).st_mode & 07777
+ newmode = (oldmode | 0555) & 07777
+ os.chmod(fn, newmode)
+ logger.info('Changed mode of %s to %s', fn, oct(newmode))
+
+def _find_file(filename, dirs):
+ for dir in dirs:
+ if os.path.exists(join(dir, filename)):
+ return join(dir, filename)
+ return filename
+
+def _install_req(py_executable, unzip=False, distribute=False):
+ if not distribute:
+ setup_fn = 'setuptools-0.6c11-py%s.egg' % sys.version[:3]
+ project_name = 'setuptools'
+ bootstrap_script = EZ_SETUP_PY
+ source = None
+ else:
+ setup_fn = None
+ source = 'distribute-0.6.8.tar.gz'
+ project_name = 'distribute'
+ bootstrap_script = DISTRIBUTE_SETUP_PY
+ try:
+ # check if the global Python has distribute installed or plain
+ # setuptools
+ import pkg_resources
+ if not hasattr(pkg_resources, '_distribute'):
+ location = os.path.dirname(pkg_resources.__file__)
+ logger.notify("A globally installed setuptools was found (in %s)" % location)
+ logger.notify("Use the --no-site-packages option to use distribute in "
+ "the virtualenv.")
+ except ImportError:
+ pass
+
+ search_dirs = file_search_dirs()
+
+ if setup_fn is not None:
+ setup_fn = _find_file(setup_fn, search_dirs)
+
+ if source is not None:
+ source = _find_file(source, search_dirs)
+
+ if is_jython and os._name == 'nt':
+ # Jython's .bat sys.executable can't handle a command line
+ # argument with newlines
+ import tempfile
+ fd, ez_setup = tempfile.mkstemp('.py')
+ os.write(fd, bootstrap_script)
+ os.close(fd)
+ cmd = [py_executable, ez_setup]
+ else:
+ cmd = [py_executable, '-c', bootstrap_script]
+ if unzip:
+ cmd.append('--always-unzip')
+ env = {}
+ if logger.stdout_level_matches(logger.DEBUG):
+ cmd.append('-v')
+
+ old_chdir = os.getcwd()
+ if setup_fn is not None and os.path.exists(setup_fn):
+ logger.info('Using existing %s egg: %s' % (project_name, setup_fn))
+ cmd.append(setup_fn)
+ if os.environ.get('PYTHONPATH'):
+ env['PYTHONPATH'] = setup_fn + os.path.pathsep + os.environ['PYTHONPATH']
+ else:
+ env['PYTHONPATH'] = setup_fn
+ else:
+ # the source is found, let's chdir
+ if source is not None and os.path.exists(source):
+ os.chdir(os.path.dirname(source))
+ else:
+ logger.info('No %s egg found; downloading' % project_name)
+ cmd.extend(['--always-copy', '-U', project_name])
+ logger.start_progress('Installing %s...' % project_name)
+ logger.indent += 2
+ cwd = None
+ if project_name == 'distribute':
+ env['DONT_PATCH_SETUPTOOLS'] = 'true'
+
+ def _filter_ez_setup(line):
+ return filter_ez_setup(line, project_name)
+
+ if not os.access(os.getcwd(), os.W_OK):
+ cwd = '/tmp'
+ if source is not None and os.path.exists(source):
+ # the current working dir is hostile, let's copy the
+ # tarball to /tmp
+ target = os.path.join(cwd, os.path.split(source)[-1])
+ shutil.copy(source, target)
+ try:
+ call_subprocess(cmd, show_stdout=False,
+ filter_stdout=_filter_ez_setup,
+ extra_env=env,
+ cwd=cwd)
+ finally:
+ logger.indent -= 2
+ logger.end_progress()
+ if os.getcwd() != old_chdir:
+ os.chdir(old_chdir)
+ if is_jython and os._name == 'nt':
+ os.remove(ez_setup)
+
+def file_search_dirs():
+ here = os.path.dirname(os.path.abspath(__file__))
+ dirs = ['.', here,
+ join(here, 'virtualenv_support')]
+ if os.path.splitext(os.path.dirname(__file__))[0] != 'virtualenv':
+ # Probably some boot script; just in case virtualenv is installed...
+ try:
+ import virtualenv
+ except ImportError:
+ pass
+ else:
+ dirs.append(os.path.join(os.path.dirname(virtualenv.__file__), 'virtualenv_support'))
+ return [d for d in dirs if os.path.isdir(d)]
+
+def install_setuptools(py_executable, unzip=False):
+ _install_req(py_executable, unzip)
+
+def install_distribute(py_executable, unzip=False):
+ _install_req(py_executable, unzip, distribute=True)
+
+_pip_re = re.compile(r'^pip-.*(zip|tar.gz|tar.bz2|tgz|tbz)$', re.I)
+def install_pip(py_executable):
+ filenames = []
+ for dir in file_search_dirs():
+ filenames.extend([join(dir, fn) for fn in os.listdir(dir)
+ if _pip_re.search(fn)])
+ filenames.sort(key=lambda x: os.path.basename(x).lower())
+ if not filenames:
+ filename = 'pip'
+ else:
+ filename = filenames[-1]
+ easy_install_script = 'easy_install'
+ if sys.platform == 'win32':
+ easy_install_script = 'easy_install-script.py'
+ cmd = [py_executable, join(os.path.dirname(py_executable), easy_install_script), filename]
+ if filename == 'pip':
+ logger.info('Installing pip from network...')
+ else:
+ logger.info('Installing %s' % os.path.basename(filename))
+ logger.indent += 2
+ def _filter_setup(line):
+ return filter_ez_setup(line, 'pip')
+ try:
+ call_subprocess(cmd, show_stdout=False,
+ filter_stdout=_filter_setup)
+ finally:
+ logger.indent -= 2
+
+def filter_ez_setup(line, project_name='setuptools'):
+ if not line.strip():
+ return Logger.DEBUG
+ if project_name == 'distribute':
+ for prefix in ('Extracting', 'Now working', 'Installing', 'Before',
+ 'Scanning', 'Setuptools', 'Egg', 'Already',
+ 'running', 'writing', 'reading', 'installing',
+ 'creating', 'copying', 'byte-compiling', 'removing',
+ 'Processing'):
+ if line.startswith(prefix):
+ return Logger.DEBUG
+ return Logger.DEBUG
+ for prefix in ['Reading ', 'Best match', 'Processing setuptools',
+ 'Copying setuptools', 'Adding setuptools',
+ 'Installing ', 'Installed ']:
+ if line.startswith(prefix):
+ return Logger.DEBUG
+ return Logger.INFO
+
+def main():
+ parser = optparse.OptionParser(
+ version=virtualenv_version,
+ usage="%prog [OPTIONS] DEST_DIR")
+
+ parser.add_option(
+ '-v', '--verbose',
+ action='count',
+ dest='verbose',
+ default=0,
+ help="Increase verbosity")
+
+ parser.add_option(
+ '-q', '--quiet',
+ action='count',
+ dest='quiet',
+ default=0,
+ help='Decrease verbosity')
+
+ parser.add_option(
+ '-p', '--python',
+ dest='python',
+ metavar='PYTHON_EXE',
+ help='The Python interpreter to use, e.g., --python=python2.5 will use the python2.5 '
+ 'interpreter to create the new environment. The default is the interpreter that '
+ 'virtualenv was installed with (%s)' % sys.executable)
+
+ parser.add_option(
+ '--clear',
+ dest='clear',
+ action='store_true',
+ help="Clear out the non-root install and start from scratch")
+
+ parser.add_option(
+ '--no-site-packages',
+ dest='no_site_packages',
+ action='store_true',
+ help="Don't give access to the global site-packages dir to the "
+ "virtual environment")
+
+ parser.add_option(
+ '--unzip-setuptools',
+ dest='unzip_setuptools',
+ action='store_true',
+ help="Unzip Setuptools or Distribute when installing it")
+
+ parser.add_option(
+ '--relocatable',
+ dest='relocatable',
+ action='store_true',
+ help='Make an EXISTING virtualenv environment relocatable. '
+ 'This fixes up scripts and makes all .pth files relative')
+
+ parser.add_option(
+ '--distribute',
+ dest='use_distribute',
+ action='store_true',
+ help="Use Distribute instead of Setuptools")
+
+ if 'extend_parser' in globals():
+ extend_parser(parser)
+
+ options, args = parser.parse_args()
+
+ global logger
+
+ if 'adjust_options' in globals():
+ adjust_options(options, args)
+
+ verbosity = options.verbose - options.quiet
+ logger = Logger([(Logger.level_for_integer(2-verbosity), sys.stdout)])
+
+ if options.python and not os.environ.get('VIRTUALENV_INTERPRETER_RUNNING'):
+ env = os.environ.copy()
+ interpreter = resolve_interpreter(options.python)
+ if interpreter == sys.executable:
+ logger.warn('Already using interpreter %s' % interpreter)
+ else:
+ logger.notify('Running virtualenv with interpreter %s' % interpreter)
+ env['VIRTUALENV_INTERPRETER_RUNNING'] = 'true'
+ file = __file__
+ if file.endswith('.pyc'):
+ file = file[:-1]
+ os.execvpe(interpreter, [interpreter, file] + sys.argv[1:], env)
+
+ if not args:
+ print 'You must provide a DEST_DIR'
+ parser.print_help()
+ sys.exit(2)
+ if len(args) > 1:
+ print 'There must be only one argument: DEST_DIR (you gave %s)' % (
+ ' '.join(args))
+ parser.print_help()
+ sys.exit(2)
+
+ home_dir = args[0]
+
+ if os.environ.get('WORKING_ENV'):
+ logger.fatal('ERROR: you cannot run virtualenv while in a workingenv')
+ logger.fatal('Please deactivate your workingenv, then re-run this script')
+ sys.exit(3)
+
+ if os.environ.get('PYTHONHOME'):
+ if sys.platform == 'win32':
+ name = '%PYTHONHOME%'
+ else:
+ name = '$PYTHONHOME'
+ logger.warn('%s is set; this can cause problems creating environments' % name)
+
+ if options.relocatable:
+ make_environment_relocatable(home_dir)
+ return
+
+ create_environment(home_dir, site_packages=not options.no_site_packages, clear=options.clear,
+ unzip_setuptools=options.unzip_setuptools,
+ use_distribute=options.use_distribute)
+ if 'after_install' in globals():
+ after_install(options, home_dir)
+
+def call_subprocess(cmd, show_stdout=True,
+ filter_stdout=None, cwd=None,
+ raise_on_returncode=True, extra_env=None):
+ cmd_parts = []
+ for part in cmd:
+ if len(part) > 40:
+ part = part[:30]+"..."+part[-5:]
+ if ' ' in part or '\n' in part or '"' in part or "'" in part:
+ part = '"%s"' % part.replace('"', '\\"')
+ cmd_parts.append(part)
+ cmd_desc = ' '.join(cmd_parts)
+ if show_stdout:
+ stdout = None
+ else:
+ stdout = subprocess.PIPE
+ logger.debug("Running command %s" % cmd_desc)
+ if extra_env:
+ env = os.environ.copy()
+ env.update(extra_env)
+ else:
+ env = None
+ try:
+ proc = subprocess.Popen(
+ cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
+ cwd=cwd, env=env)
+ except Exception, e:
+ logger.fatal(
+ "Error %s while executing command %s" % (e, cmd_desc))
+ raise
+ all_output = []
+ if stdout is not None:
+ stdout = proc.stdout
+ while 1:
+ line = stdout.readline()
+ if not line:
+ break
+ line = line.rstrip()
+ all_output.append(line)
+ if filter_stdout:
+ level = filter_stdout(line)
+ if isinstance(level, tuple):
+ level, line = level
+ logger.log(level, line)
+ if not logger.stdout_level_matches(level):
+ logger.show_progress()
+ else:
+ logger.info(line)
+ else:
+ proc.communicate()
+ proc.wait()
+ if proc.returncode:
+ if raise_on_returncode:
+ if all_output:
+ logger.notify('Complete output from command %s:' % cmd_desc)
+ logger.notify('\n'.join(all_output) + '\n----------------------------------------')
+ raise OSError(
+ "Command %s failed with error code %s"
+ % (cmd_desc, proc.returncode))
+ else:
+ logger.warn(
+ "Command %s had error code %s"
+ % (cmd_desc, proc.returncode))
+
+
+def create_environment(home_dir, site_packages=True, clear=False,
+ unzip_setuptools=False, use_distribute=False):
+ """
+ Creates a new environment in ``home_dir``.
+
+ If ``site_packages`` is true (the default) then the global
+ ``site-packages/`` directory will be on the path.
+
+ If ``clear`` is true (default False) then the environment will
+ first be cleared.
+ """
+ home_dir, lib_dir, inc_dir, bin_dir = path_locations(home_dir)
+
+ py_executable = os.path.abspath(install_python(
+ home_dir, lib_dir, inc_dir, bin_dir,
+ site_packages=site_packages, clear=clear))
+
+ install_distutils(lib_dir, home_dir)
+
+ if use_distribute:
+ install_distribute(py_executable, unzip=unzip_setuptools)
+ else:
+ install_setuptools(py_executable, unzip=unzip_setuptools)
+
+ install_pip(py_executable)
+
+ install_activate(home_dir, bin_dir)
+
+def path_locations(home_dir):
+ """Return the path locations for the environment (where libraries are,
+ where scripts go, etc)"""
+ # XXX: We'd use distutils.sysconfig.get_python_inc/lib but its
+ # prefix arg is broken: http://bugs.python.org/issue3386
+ if sys.platform == 'win32':
+ # Windows has lots of problems with executables with spaces in
+ # the name; this function will remove them (using the ~1
+ # format):
+ mkdir(home_dir)
+ if ' ' in home_dir:
+ try:
+ import win32api
+ except ImportError:
+ print 'Error: the path "%s" has a space in it' % home_dir
+ print 'To handle these kinds of paths, the win32api module must be installed:'
+ print ' http://sourceforge.net/projects/pywin32/'
+ sys.exit(3)
+ home_dir = win32api.GetShortPathName(home_dir)
+ lib_dir = join(home_dir, 'Lib')
+ inc_dir = join(home_dir, 'Include')
+ bin_dir = join(home_dir, 'Scripts')
+ elif is_jython:
+ lib_dir = join(home_dir, 'Lib')
+ inc_dir = join(home_dir, 'Include')
+ bin_dir = join(home_dir, 'bin')
+ else:
+ lib_dir = join(home_dir, 'lib', py_version)
+ inc_dir = join(home_dir, 'include', py_version)
+ bin_dir = join(home_dir, 'bin')
+ return home_dir, lib_dir, inc_dir, bin_dir
+
+def install_python(home_dir, lib_dir, inc_dir, bin_dir, site_packages, clear):
+ """Install just the base environment, no distutils patches etc"""
+ if sys.executable.startswith(bin_dir):
+ print 'Please use the *system* python to run this script'
+ return
+
+ if clear:
+ rmtree(lib_dir)
+ ## FIXME: why not delete it?
+ ## Maybe it should delete everything with #!/path/to/venv/python in it
+ logger.notify('Not deleting %s', bin_dir)
+
+ if hasattr(sys, 'real_prefix'):
+ logger.notify('Using real prefix %r' % sys.real_prefix)
+ prefix = sys.real_prefix
+ else:
+ prefix = sys.prefix
+ mkdir(lib_dir)
+ fix_lib64(lib_dir)
+ stdlib_dirs = [os.path.dirname(os.__file__)]
+ if sys.platform == 'win32':
+ stdlib_dirs.append(join(os.path.dirname(stdlib_dirs[0]), 'DLLs'))
+ elif sys.platform == 'darwin':
+ stdlib_dirs.append(join(stdlib_dirs[0], 'site-packages'))
+ for stdlib_dir in stdlib_dirs:
+ if not os.path.isdir(stdlib_dir):
+ continue
+ if hasattr(os, 'symlink'):
+ logger.info('Symlinking Python bootstrap modules')
+ else:
+ logger.info('Copying Python bootstrap modules')
+ logger.indent += 2
+ try:
+ for fn in os.listdir(stdlib_dir):
+ if fn != 'site-packages' and os.path.splitext(fn)[0] in REQUIRED_MODULES:
+ copyfile(join(stdlib_dir, fn), join(lib_dir, fn))
+ finally:
+ logger.indent -= 2
+ mkdir(join(lib_dir, 'site-packages'))
+ writefile(join(lib_dir, 'site.py'), SITE_PY)
+ writefile(join(lib_dir, 'orig-prefix.txt'), prefix)
+ site_packages_filename = join(lib_dir, 'no-global-site-packages.txt')
+ if not site_packages:
+ writefile(site_packages_filename, '')
+ else:
+ if os.path.exists(site_packages_filename):
+ logger.info('Deleting %s' % site_packages_filename)
+ os.unlink(site_packages_filename)
+
+ stdinc_dir = join(prefix, 'include', py_version)
+ if os.path.exists(stdinc_dir):
+ copyfile(stdinc_dir, inc_dir)
+ else:
+ logger.debug('No include dir %s' % stdinc_dir)
+
+ if sys.exec_prefix != prefix:
+ if sys.platform == 'win32':
+ exec_dir = join(sys.exec_prefix, 'lib')
+ elif is_jython:
+ exec_dir = join(sys.exec_prefix, 'Lib')
+ else:
+ exec_dir = join(sys.exec_prefix, 'lib', py_version)
+ for fn in os.listdir(exec_dir):
+ copyfile(join(exec_dir, fn), join(lib_dir, fn))
+
+ if is_jython:
+ # Jython has either jython-dev.jar and javalib/ dir, or just
+ # jython.jar
+ for name in 'jython-dev.jar', 'javalib', 'jython.jar':
+ src = join(prefix, name)
+ if os.path.exists(src):
+ copyfile(src, join(home_dir, name))
+ # XXX: registry should always exist after Jython 2.5rc1
+ src = join(prefix, 'registry')
+ if os.path.exists(src):
+ copyfile(src, join(home_dir, 'registry'), symlink=False)
+ copyfile(join(prefix, 'cachedir'), join(home_dir, 'cachedir'),
+ symlink=False)
+
+ mkdir(bin_dir)
+ py_executable = join(bin_dir, os.path.basename(sys.executable))
+ if 'Python.framework' in prefix:
+ if py_executable.endswith('/Python'):
+ # The name of the python executable is not quite what
+ # we want, rename it.
+ py_executable = os.path.join(
+ os.path.dirname(py_executable), 'python')
+
+ logger.notify('New %s executable in %s', expected_exe, py_executable)
+ if sys.executable != py_executable:
+ ## FIXME: could I just hard link?
+ executable = sys.executable
+ if sys.platform == 'cygwin' and os.path.exists(executable + '.exe'):
+ # Cygwin misreports sys.executable sometimes
+ executable += '.exe'
+ py_executable += '.exe'
+ logger.info('Executable actually exists in %s' % executable)
+ shutil.copyfile(executable, py_executable)
+ make_exe(py_executable)
+ if sys.platform == 'win32' or sys.platform == 'cygwin':
+ pythonw = os.path.join(os.path.dirname(sys.executable), 'pythonw.exe')
+ if os.path.exists(pythonw):
+ logger.info('Also created pythonw.exe')
+ shutil.copyfile(pythonw, os.path.join(os.path.dirname(py_executable), 'pythonw.exe'))
+
+ if os.path.splitext(os.path.basename(py_executable))[0] != expected_exe:
+ secondary_exe = os.path.join(os.path.dirname(py_executable),
+ expected_exe)
+ py_executable_ext = os.path.splitext(py_executable)[1]
+ if py_executable_ext == '.exe':
+ # python2.4 gives an extension of '.4' :P
+ secondary_exe += py_executable_ext
+ if os.path.exists(secondary_exe):
+ logger.warn(