-
Notifications
You must be signed in to change notification settings - Fork 353
/
database.py
137 lines (106 loc) · 4.02 KB
/
database.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
# Copyright 2009 Google Inc. Released under the GPL v2
# This file contains the classes used for the known kernel versions persistent
# storage
import cPickle, fcntl, os, tempfile
class item(object):
"""Wrap a file item stored in a database."""
def __init__(self, name, size, timestamp):
assert type(size) == int
assert type(timestamp) == int
self.name = name
self.size = size
self.timestamp = timestamp
def __repr__(self):
return ("database.item('%s', %d, %d)" %
(self.name, self.size, self.timestamp))
def __eq__(self, other):
if not isinstance(other, item):
return NotImplementedError
return (self.name == other.name and self.size == other.size and
self.timestamp == other.timestamp)
def __ne__(self, other):
return not self.__eq__(other)
class database(object):
"""
This is an Abstract Base Class for the file items database, not strictly
needed in Python because of the dynamic nature of the language but useful
to document the expected common API of the implementations.
"""
def get_dictionary(self):
"""
Should be implemented to open and read the persistent contents of
the database and return it as a key->value dictionary.
"""
raise NotImplementedError('get_dictionary not implemented')
def merge_dictionary(self, values):
"""
Should be implemented to merge the "values" dictionary into the
database persistent contents (ie to update existent entries and to add
those that do not exist).
"""
raise NotImplementedError('merge_dictionary not implemented')
class dict_database(database):
"""
A simple key->value database that uses standard python pickle dump of
a dictionary object for persistent storage.
"""
def __init__(self, path):
self.path = path
def get_dictionary(self, _open_func=open):
"""
Return the key/value pairs as a standard dictionary.
"""
try:
fd = _open_func(self.path, 'rb')
except IOError:
# no db file, considering as if empty dictionary
res = {}
else:
try:
res = cPickle.load(fd)
finally:
fd.close()
return res
def _aquire_lock(self):
fd = os.open(self.path + '.lock', os.O_RDONLY | os.O_CREAT)
try:
# this may block
fcntl.flock(fd, fcntl.LOCK_EX)
except Exception, err:
os.close(fd)
raise err
return fd
def merge_dictionary(self, values):
"""
Merge the contents of "values" with the current contents of the
database.
"""
if not values:
return
# use file locking to make the read/write of the file atomic
lock_fd = self._aquire_lock()
# make sure we release locks in case of exceptions (in case the
# process dies the OS will release them for us)
try:
contents = self.get_dictionary()
contents.update(values)
# use a tempfile/atomic-rename technique to not require
# synchronization for get_dictionary() calls and also protect
# against full disk file corruption situations
fd, fname = tempfile.mkstemp(prefix=os.path.basename(self.path),
dir=os.path.dirname(self.path))
write_file = os.fdopen(fd, 'wb')
try:
try:
cPickle.dump(contents, write_file,
protocol=cPickle.HIGHEST_PROTOCOL)
finally:
write_file.close()
# this is supposed to be atomic on POSIX systems
os.rename(fname, self.path)
except Exception:
os.unlink(fname)
raise
finally:
# close() releases any locks on that fd
os.close(lock_fd)