Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 254 lines (183 sloc) 7.496 kb
767facb8 » lirazsiri
2012-08-08 cleanup whitespace
1 #
7930886c » lirazsiri
2013-08-28 major refactoring
2 # Copyright (c) 2010-2013 Liraz Siri <liraz@turnkeylinux.org>
767facb8 » lirazsiri
2012-08-08 cleanup whitespace
3 #
0c11ee92 » lirazsiri
2010-08-10 added copyright notices to everything
4 # This file is part of TKLBAM (TurnKey Linux BAckup and Migration).
767facb8 » lirazsiri
2012-08-08 cleanup whitespace
5 #
e34925a4 » lirazsiri
2010-08-11 free software => open source software
6 # TKLBAM is open source software; you can redistribute it and/or
0c11ee92 » lirazsiri
2010-08-10 added copyright notices to everything
7 # modify it under the terms of the GNU General Public License as
8 # published by the Free Software Foundation; either version 3 of
9 # the License, or (at your option) any later version.
767facb8 » lirazsiri
2012-08-08 cleanup whitespace
10 #
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
11 import os
0a7f0ca6 » lirazsiri
2010-08-19 patched tklbam to use tklbam-duplicity and tklbam-python-boto
12 from os.path import *
13
7765409a » lirazsiri
2010-08-08 cleanup before backing up (disables checkpoint/restore)
14 import sys
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
15
16 from subprocess import *
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
17 from squid import Squid
18
7930886c » lirazsiri
2013-08-28 major refactoring
19 from utils import AttrDict
20
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
21 import resource
22 RLIMIT_NOFILE_MAX = 8192
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
23
0a7f0ca6 » lirazsiri
2010-08-19 patched tklbam to use tklbam-duplicity and tklbam-python-boto
24 def _find_duplicity_pylib(path):
25 if not isdir(path):
26 return None
27
28 for fpath, dnames, fnames in os.walk(path):
29 if 'duplicity' in dnames:
30 return fpath
31
32 return None
33
34 PATH_DEPS = os.environ.get('TKLBAM_DEPS', '/usr/lib/tklbam/deps')
35 PATH_DEPS_BIN = join(PATH_DEPS, "bin")
36 PATH_DEPS_PYLIB = _find_duplicity_pylib(PATH_DEPS)
37
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
38 class Error(Exception):
39 pass
40
7930886c » lirazsiri
2013-08-28 major refactoring
41 class Duplicity:
42 """low-level interface to Duplicity"""
43
1b0c7d18 » lirazsiri
2010-08-05 improved duplicity.Command interface
44 def __init__(self, *args):
45 """Duplicity command. The first member of args can be a an array of tuple arguments"""
46
47 if isinstance(args[0], list):
d30933a1 » lirazsiri
2013-08-29 bugfix: copy opts otherwise archive-dir is replicated
48 opts = args[0][:]
1b0c7d18 » lirazsiri
2010-08-05 improved duplicity.Command interface
49 args = args[1:]
50 else:
51 opts = []
52
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
53 if not args:
54 raise Error("no arguments!")
55
56 opts += [ ('archive-dir', '/var/cache/duplicity') ]
57
58 opts = [ "--%s=%s" % (key, val) for key, val in opts ]
59 self.command = ["duplicity"] + opts + list(args)
60
1b0f87c0 » lirazsiri
2012-08-07 tklbam-backup --debug command
61 def run(self, passphrase, creds=None, debug=False):
7765409a » lirazsiri
2010-08-08 cleanup before backing up (disables checkpoint/restore)
62 sys.stdout.flush()
63
2cf23fbb » lirazsiri
2010-08-09 pass S3 credentials to boto + refactored credentials into a class
64 if creds:
65 os.environ['AWS_ACCESS_KEY_ID'] = creds.accesskey
66 os.environ['AWS_SECRET_ACCESS_KEY'] = creds.secretkey
f43537a9 » lirazsiri
2012-08-06 tklbam now depends on new tklbam-python-boto version
67 os.environ['X_AMZ_SECURITY_TOKEN'] = ",".join([creds.producttoken, creds.usertoken])
2cf23fbb » lirazsiri
2010-08-09 pass S3 credentials to boto + refactored credentials into a class
68
0a7f0ca6 » lirazsiri
2010-08-19 patched tklbam to use tklbam-duplicity and tklbam-python-boto
69 if PATH_DEPS_BIN not in os.environ['PATH'].split(':'):
70 os.environ['PATH'] = PATH_DEPS_BIN + ':' + os.environ['PATH']
71
72 if PATH_DEPS_PYLIB:
73 os.environ['PYTHONPATH'] = PATH_DEPS_PYLIB
74
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
75 os.environ['PASSPHRASE'] = passphrase
f43537a9 » lirazsiri
2012-08-06 tklbam now depends on new tklbam-python-boto version
76
1b0f87c0 » lirazsiri
2012-08-07 tklbam-backup --debug command
77 if debug:
39f7ade9 » lirazsiri
2013-10-23 restore: --debug launched pre-Duplicity shell + better explain what t…
78 print """
79 The --debug option has dropped you into an interactive shell in which you can
80 explore the state of the system just before the above duplicity command is
81 run, and/or execute it manually.
82
83 For Duplicity usage info, options and storage backends, run "duplicity --help".
84 To exit from the shell and continue running duplicity "exit 0".
85 To exit from the shell and abort this session "exit 1".
86 """
87
1b0f87c0 » lirazsiri
2012-08-07 tklbam-backup --debug command
88 import executil
d928e2f5 » lirazsiri
2013-10-11 bugfix: tklbam-backup --debug should execute bash with --norc
89 shell = os.environ.get("SHELL", "/bin/bash")
90 if shell == "/bin/bash":
91 shell += " --norc"
54026efd » lirazsiri
2013-10-22 tklbam-backup: print help instructions before executing --debug shell
92
d928e2f5 » lirazsiri
2013-10-11 bugfix: tklbam-backup --debug should execute bash with --norc
93 executil.system(shell)
54026efd » lirazsiri
2013-10-22 tklbam-backup: print help instructions before executing --debug shell
94
1b0f87c0 » lirazsiri
2012-08-07 tklbam-backup --debug command
95
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
96 child = Popen(self.command)
97 del os.environ['PASSPHRASE']
98
99 exitcode = child.wait()
100 if exitcode != 0:
101 raise Error("non-zero exitcode (%d) from backup command: %s" % (exitcode, str(self)))
767facb8 » lirazsiri
2012-08-08 cleanup whitespace
102
82507075 » lirazsiri
2010-08-05 refactored out duplicity execution to a separate module
103 def __str__(self):
104 return " ".join(self.command)
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
105
106
107 def _raise_rlimit(type, newlimit):
108 soft, hard = resource.getrlimit(type)
109 if soft > newlimit:
110 return
111
112 if hard > newlimit:
113 return resource.setrlimit(type, (newlimit, hard))
114
115 try:
116 resource.setrlimit(type, (newlimit, newlimit))
117 except ValueError:
118 return
119
7930886c » lirazsiri
2013-08-28 major refactoring
120 class Target(AttrDict):
121 def __init__(self, address, credentials, secret):
122 AttrDict.__init__(self)
123 self.address = address
124 self.credentials = credentials
125 self.secret = secret
126
127 class Downloader(AttrDict):
128 """High-level interface to Duplicity downloads"""
129
130 CACHE_SIZE = "50%"
131 CACHE_DIR = "/var/cache/tklbam/restore"
132
133 def __init__(self, time=None, cache_size=CACHE_SIZE, cache_dir=CACHE_DIR):
134 AttrDict.__init__(self)
135
136 self.time = time
137 self.cache_size = cache_size
138 self.cache_dir = cache_dir
139
50ad6f89 » lirazsiri
2013-11-07 tklbam-restore: --force forces --raw-download to work with non-empty …
140 def __call__(self, download_path, target, debug=False, log=None, force=False):
39f7ade9 » lirazsiri
2013-10-23 restore: --debug launched pre-Duplicity shell + better explain what t…
141 if log is None:
142 log = lambda s: None
143
7930886c » lirazsiri
2013-08-28 major refactoring
144 if self.time:
145 opts = [("restore-time", self.time)]
146 else:
147 opts = []
148
4a009be8 » lirazsiri
2013-10-24 refined backup output to be more consistent with restore output
149 log("// started squid: caching downloaded backup archives to " + self.cache_dir + "\n")
39f7ade9 » lirazsiri
2013-10-23 restore: --debug launched pre-Duplicity shell + better explain what t…
150
7930886c » lirazsiri
2013-08-28 major refactoring
151 squid = Squid(self.cache_size, self.cache_dir)
152 squid.start()
153
154 orig_env = os.environ.get('http_proxy')
155 os.environ['http_proxy'] = squid.address
156
157 _raise_rlimit(resource.RLIMIT_NOFILE, RLIMIT_NOFILE_MAX)
50ad6f89 » lirazsiri
2013-11-07 tklbam-restore: --force forces --raw-download to work with non-empty …
158 args = [ '--s3-unencrypted-connection', target.address, download_path ]
159 if force:
160 args = [ '--force' ] + args
161
162 command = Duplicity(opts, *args)
39f7ade9 » lirazsiri
2013-10-23 restore: --debug launched pre-Duplicity shell + better explain what t…
163
164 log("# " + str(command))
165
166 command.run(target.secret, target.credentials, debug=debug)
7930886c » lirazsiri
2013-08-28 major refactoring
167
168 if orig_env:
169 os.environ['http_proxy'] = orig_env
170 else:
171 del os.environ['http_proxy']
172
173 sys.stdout.flush()
174
4a009be8 » lirazsiri
2013-10-24 refined backup output to be more consistent with restore output
175 log("\n// stopping squid: download complete so caching no longer required\n")
7930886c » lirazsiri
2013-08-28 major refactoring
176 squid.stop()
177
178 class Uploader(AttrDict):
179 """High-level interface to Duplicity uploads"""
180
181 VOLSIZE = 25
182 FULL_IF_OLDER_THAN = "1M"
183 S3_PARALLEL_UPLOADS = 1
184
185 def __init__(self,
186 verbose=True,
187 volsize=VOLSIZE,
188 full_if_older_than=FULL_IF_OLDER_THAN,
189 s3_parallel_uploads=S3_PARALLEL_UPLOADS,
190
191 includes=[],
192 include_filelist=None,
193 excludes=[],
194 ):
195
196 AttrDict.__init__(self)
197
198 self.verbose = verbose
199 self.volsize = volsize
200 self.full_if_older_than = full_if_older_than
201 self.s3_parallel_uploads = s3_parallel_uploads
202
203 self.includes = includes
204 self.include_filelist = include_filelist
205 self.excludes = excludes
206
207 def __call__(self, source_dir, target, force_cleanup=True, dry_run=False, debug=False, log=None):
208 if log is None:
209 log = lambda s: None
210
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
211 opts = []
7930886c » lirazsiri
2013-08-28 major refactoring
212 if self.verbose:
213 opts += [('verbosity', 5)]
214
215 if force_cleanup:
216 cleanup_command = Duplicity(opts, "cleanup", "--force", target.address)
217 log(cleanup_command)
218
219 if not dry_run:
220 cleanup_command.run(target.secret, target.credentials)
221
84ea8245 » lirazsiri
2013-10-21 tklbam-backup UX: make backup verbose and transparent
222 log("\n")
223
7930886c » lirazsiri
2013-08-28 major refactoring
224 opts += [('volsize', self.volsize),
225 ('full-if-older-than', self.full_if_older_than),
226 ('gpg-options', '--cipher-algo=aes')]
227
228 for include in self.includes:
229 opts += [ ('include', include) ]
230
231 if self.include_filelist:
232 opts += [ ('include-filelist', self.include_filelist) ]
233
234 for exclude in self.excludes:
235 opts += [ ('exclude', exclude) ]
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
236
7930886c » lirazsiri
2013-08-28 major refactoring
237 args = [ '--s3-unencrypted-connection', '--allow-source-mismatch' ]
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
238
7930886c » lirazsiri
2013-08-28 major refactoring
239 if dry_run:
240 args += [ '--dry-run' ]
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
241
7930886c » lirazsiri
2013-08-28 major refactoring
242 if self.s3_parallel_uploads > 1:
243 s3_multipart_chunk_size = self.volsize / self.s3_parallel_uploads
244 if s3_multipart_chunk_size < 5:
245 s3_multipart_chunk_size = 5
246 args += [ '--s3-use-multiprocessing', '--s3-multipart-chunk-size=%d' % s3_multipart_chunk_size ]
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
247
7930886c » lirazsiri
2013-08-28 major refactoring
248 args += [ source_dir, target.address ]
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
249
7930886c » lirazsiri
2013-08-28 major refactoring
250 backup_command = Duplicity(opts, *args)
5909a174 » lirazsiri
2013-08-26 refactor duplicity restore logic out of restore module and into dupli…
251
84ea8245 » lirazsiri
2013-10-21 tklbam-backup UX: make backup verbose and transparent
252 log(str(backup_command))
7930886c » lirazsiri
2013-08-28 major refactoring
253 backup_command.run(target.secret, target.credentials, debug=debug)
84ea8245 » lirazsiri
2013-10-21 tklbam-backup UX: make backup verbose and transparent
254 log("\n")
Something went wrong with that request. Please try again.