Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 351 lines (258 sloc) 10.009 kb
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
1 import os
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
2 from os.path import exists, join
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
3
4 import re
5 import struct
6 import base64
7 from hashlib import sha1 as sha
8 from paths import Paths
9 import pickle
10 import glob
11 from datetime import datetime
12
13 from utils import AttrDict
14
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
15 from hub import Credentials, ProfileArchive
676ec06 @lirazsiri refactoring: NotSubscribedError => NotSubscribed
lirazsiri authored
16 from hub import Error, NotSubscribed, InvalidBackupError
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
17
18 class APIKey:
19 def __init__(self, apikey):
20 apikey = str(apikey)
21 self.encoded = apikey
767facb @lirazsiri cleanup whitespace
lirazsiri authored
22
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
23 padded = "A" * (20 - len(apikey)) + apikey
24 try:
25 uid, secret = struct.unpack("!L8s", base64.b32decode(padded + "=" * 4))
26 except TypeError:
27 raise Error("Invalid characters in API-KEY")
28
29 self.uid = uid
30 self.secret = secret
31
32 @classmethod
33 def generate(cls, uid, secret=None):
34 if secret is None:
35 secret = os.urandom(8)
36 else:
37 secret = sha(secret).digest()[:8]
38
39 packed = struct.pack("!L8s", uid, secret)
40 encoded = base64.b32encode(packed).lstrip("A").rstrip("=")
41
42 return cls(encoded)
43
44 def subkey(self, namespace):
45 return self.generate(self.uid, namespace + self.secret)
46
47 def __str__(self):
48 return self.encoded
49
50 def __repr__(self):
51 return "APIKey(%s)" % `str(self)`
52
53 def __eq__(self, other):
54 return self.encoded == other.encoded
55
56 def __ne__(self, other):
57 return not self.__eq__(other)
58
59 class DummyUser(AttrDict):
60 def __init__(self, uid, apikey):
61 self.uid = uid
62 self.apikey = apikey
63 self.credentials = None
64 self.backups = {}
65 self.backups_max = 0
66
67 def subscribe(self):
68 accesskey = base64.b64encode(sha("%d" % self.uid).digest())[:20]
69 secretkey = base64.b64encode(os.urandom(30))[:40]
8996ce8 @lirazsiri update credentials interface to match real hub
lirazsiri authored
70 producttoken = "{ProductToken}" + base64.b64encode("\x00" + os.urandom(2) + "AppTkn" + os.urandom(224))
71 usertoken = "{UserToken}" + base64.b64encode("\x00" + os.urandom(2) + "UserTkn" + os.urandom(288))
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
72
767facb @lirazsiri cleanup whitespace
lirazsiri authored
73 self.credentials = Credentials({'accesskey': accesskey,
74 'secretkey': secretkey,
8996ce8 @lirazsiri update credentials interface to match real hub
lirazsiri authored
75 'producttoken': producttoken,
76 'usertoken': usertoken})
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
77
78 def unsubscribe(self):
79 self.credentials = None
80
b666703 @lirazsiri profile_id detection for non-TurnKey systems
lirazsiri authored
81 def new_backup(self, address, key, profile_id, server_id=None):
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
82 self.backups_max += 1
83
84 id = str(self.backups_max)
85 backup_record = DummyBackupRecord(id, address, key, \
b666703 @lirazsiri profile_id detection for non-TurnKey systems
lirazsiri authored
86 profile_id, server_id)
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
87
88 self.backups[id] = backup_record
89
90 return backup_record
91
92 class DuplicityFile(AttrDict):
93 @classmethod
94 def from_fname(cls, fname):
95 m = re.match(r'duplicity-(.*?)\.(.*?).(?:sigtar|vol.*difftar)', fname)
96 if not m:
97 return None
98
99 type, timestamp = m.groups()
100 m = re.search(r'to\.(.*)', timestamp)
101 if m:
102 timestamp, = m.groups()
103
104 if 'full' in type:
105 type = 'full'
106 else:
107 type = 'inc'
108
109 try:
fadb56c @lirazsiri updated duplicity timestamp format
lirazsiri authored
110 timestamp = datetime.strptime(timestamp, "%Y%m%dT%H%M%SZ")
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
111 except ValueError:
112 return
113
114 return cls(type, timestamp)
115
116 def __init__(self, type, timestamp):
117 self.type = type
118 self.timestamp = timestamp
119
120 class DummySession(AttrDict):
121 def __init__(self, type, timestamp, size=0):
122 self.type = type
123 self.timestamp = timestamp
124 self.size = size
125
126 def _parse_duplicity_sessions(path):
127 sessions = {}
128 for fname in os.listdir(path):
129 fpath = join(path, fname)
130 fsize = os.stat(fpath).st_size
131
132 df = DuplicityFile.from_fname(fname)
133 if not df:
134 continue
135
136 if not df.timestamp in sessions:
137 sessions[df.timestamp] = DummySession(df.type, df.timestamp, fsize)
138 else:
139 sessions[df.timestamp].size += fsize
140
141 return sessions.values()
142
143 class DummyBackupRecord(AttrDict):
144 # backup_id, address
b666703 @lirazsiri profile_id detection for non-TurnKey systems
lirazsiri authored
145 def __init__(self, backup_id, address, key, profile_id, server_id):
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
146 self.backup_id = backup_id
147 self.address = address
148 self.key = key
b666703 @lirazsiri profile_id detection for non-TurnKey systems
lirazsiri authored
149 self.profile_id = profile_id
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
150 self.server_id = server_id
151
152 self.created = datetime.now()
153 self.updated = None
154
155 # in MBs
156 self.size = 0
157 self.label = "TurnKey Backup"
158
159 # no user interface for this in the dummy hub
160 self.sessions = []
161
162 def update(self):
163 self.updated = datetime.now()
164
767facb @lirazsiri cleanup whitespace
lirazsiri authored
165 path = self.address[len("file://"):]
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
166
167 self.sessions = _parse_duplicity_sessions(path)
6356cea @lirazsiri simulate Alon's breakage of dummy API specification
lirazsiri authored
168 self.size = sum([ session.size for session in self.sessions ])
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
169
170 class _DummyDB(AttrDict):
171 class Paths(Paths):
172 files = ['users', 'profiles']
173
174 @staticmethod
175 def _save(path, obj):
176 pickle.dump(obj, file(path, "w"))
177
178 @staticmethod
179 def _load(path, default=None):
180 if not exists(path):
181 return default
182
a2f48f7 @lirazsiri dummyhub bugfix: dummydb save failed due to pickle circular import
lirazsiri authored
183 try:
184 return pickle.load(file(path))
185 except:
186 return default
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
187
188 def save(self):
189 self._save(self.path.users, self.users)
190
191 def load(self):
192 self.users = self._load(self.path.users, {})
193
194 def __init__(self, path):
195 if not exists(path):
196 os.makedirs(path)
197
198 self.path = self.Paths(path)
199 self.load()
200
201 def get_user(self, uid):
202 if uid not in self.users:
203 return None
204
205 return self.users[uid]
206
207 def add_user(self):
208 if self.users:
209 uid = max(self.users.keys()) + 1
210 else:
211 uid = 1
212
213 apikey = APIKey.generate(uid)
214
215 user = DummyUser(uid, apikey)
216 self.users[uid] = user
217
218 return user
219
220
b666703 @lirazsiri profile_id detection for non-TurnKey systems
lirazsiri authored
221 def get_profile(self, profile_id):
222 matches = glob.glob("%s/%s.tar.*" % (self.path.profiles, profile_id))
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
223 if not matches:
224 return None
225
226 return matches[0]
227
a2f48f7 @lirazsiri dummyhub bugfix: dummydb save failed due to pickle circular import
lirazsiri authored
228 try:
229 dummydb
230 except NameError:
231 dummydb = _DummyDB("/var/tmp/tklbam/dummyhub")
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
232
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
233 class DummyProfileArchive(ProfileArchive):
234 def __del__(self):
235 pass
236
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
237 class Backups:
238 # For simplicity's sake this implements a dummy version of both
239 # client-side and server-side operations.
767facb @lirazsiri cleanup whitespace
lirazsiri authored
240 #
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
241 # When translating to a real implementation the interface should remain
242 # but the implementation will change completely as only client-side
243 # operations remain.
244
245 Error = Error
d7a1e9e @lirazsiri UX improvements: better usage examples for backup, restore and init
lirazsiri authored
246 class NotInitialized(Error):
247 pass
248
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
249 SUBKEY_NS = "tklbam"
250
251 @classmethod
252 def get_sub_apikey(cls, apikey):
253 """Check that APIKey is valid and return subkey"""
254 apikey = APIKey(apikey)
255 user = dummydb.get_user(apikey.uid)
256
257 if not user or user.apikey != apikey:
258 raise Error("invalid APIKey: %s" % apikey)
259
260 return apikey.subkey(cls.SUBKEY_NS)
261
262 def __init__(self, subkey):
263 if subkey is None:
ce6ce19 @lirazsiri created solo mode: allows TKLBAM to be initialized without a link to the...
lirazsiri authored
264 raise self.NotInitialized("no APIKEY - tklbam not linked to the Hub")
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
265
266 subkey = APIKey(subkey)
267
268 # the non-dummy implementation should only check the subkey when an
269 # action is performed. (I.e., NOT on initialization). In a REST API
270 # the subkey should probably be passed as an authentication header.
271
272 user = dummydb.get_user(subkey.uid)
273 if not user or subkey != user.apikey.subkey(self.SUBKEY_NS):
274 raise Error("invalid authentication subkey: %s" % subkey)
275
276 self.user = user
277
278 def get_credentials(self):
279 if not self.user.credentials:
676ec06 @lirazsiri refactoring: NotSubscribedError => NotSubscribed
lirazsiri authored
280 raise NotSubscribed()
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
281
282 return self.user.credentials
283
284 def update_key(self, backup_id, key):
285 self.get_backup_record(backup_id).key = key
286 dummydb.save()
287
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
288 def get_new_profile(self, profile_id, profile_timestamp):
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
289 """
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
290 Gets a profile for <profile_id> that is newer than <profile_timestamp>.
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
291
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
292 If there's a new profile, returns a DummyProfileArchive instance.
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
293 Otherwise returns None.
294
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
295 Raises an exception if no profile exists for profile_id.
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
296 """
297
c922ea2 @lirazsiri bugfix: fix handling of an unsubscribed Hub account in tklbam-init and r...
lirazsiri authored
298 if not self.user.credentials:
676ec06 @lirazsiri refactoring: NotSubscribedError => NotSubscribed
lirazsiri authored
299 raise NotSubscribed()
c922ea2 @lirazsiri bugfix: fix handling of an unsubscribed Hub account in tklbam-init and r...
lirazsiri authored
300
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
301 archive = dummydb.get_profile(profile_id)
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
302 if not archive:
d660d16 @lirazsiri made the dummyhub's error exception more similar to the real hub
lirazsiri authored
303 raise Error(404, 'BackupArchive.NotFound', 'Backup profile archive not found: ' + profile_id)
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
304
c58b6d8 @lirazsiri bugfix: reduce profile timestamp resolution to seconds
lirazsiri authored
305 archive_timestamp = int(os.stat(archive).st_mtime)
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
306 if profile_timestamp and profile_timestamp >= archive_timestamp:
307 return None
308
1250654 @lirazsiri allow the user to pass --profile="<profile-id>" on the cli
lirazsiri authored
309 return DummyProfileArchive(profile_id, archive, archive_timestamp)
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
310
b666703 @lirazsiri profile_id detection for non-TurnKey systems
lirazsiri authored
311 def new_backup_record(self, key, profile_id, server_id=None):
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
312 # in the real implementation the hub would create a bucket not a dir...
313 # the real implementation would have to make sure this is unique
314 path = "/var/tmp/duplicity/" + base64.b32encode(os.urandom(10))
315 os.makedirs(path)
316 address = "file://" + path
317
767facb @lirazsiri cleanup whitespace
lirazsiri authored
318 backup_record = self.user.new_backup(address, key,
b666703 @lirazsiri profile_id detection for non-TurnKey systems
lirazsiri authored
319 profile_id, server_id)
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
320
321 dummydb.save()
322
323 return backup_record
324
325 def get_backup_record(self, backup_id):
326 if backup_id not in self.user.backups:
327 raise InvalidBackupError("no such backup (%s)" % backup_id)
328
329 return self.user.backups[backup_id]
330
331 def list_backups(self):
2b2e0dc @lirazsiri updated dummy list_backups to return list of backups ordered by id
lirazsiri authored
332 backups = self.user.backups.values()
767facb @lirazsiri cleanup whitespace
lirazsiri authored
333 return sorted(self.user.backups.values(),
2b2e0dc @lirazsiri updated dummy list_backups to return list of backups ordered by id
lirazsiri authored
334 lambda a,b: cmp(int(a.backup_id), int(b.backup_id)))
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
335
336 def updated_backup(self, address):
337 # In the real implementation this should add a task which queries S3
338 # with the user's credentials and updates the Hub database (e.g., size,
339 # data on backup sessions, etc.)
340
341 for backup in self.user.backups.values():
342 if address == backup.address:
343 backup.update()
344 dummydb.save()
345 return
346
f933ff4 @lirazsiri dummyhub: set_backup_inprogress stub method
lirazsiri authored
347 def set_backup_inprogress(self, backup_id, bool):
348 pass
349
ed00b2d @lirazsiri added dummyhub support
lirazsiri authored
350
Something went wrong with that request. Please try again.