Permalink
Browse files

Change index; include atime; pack time as xstat timespec; use ns in m…

…emory.

Update the index format header to 'BUPI\0\0\0\4' (version 4).

Change the index to include atimes, and write all times to bupindex as
xstat timespecs.  This is in preparation for indexing all metadata.
After moving all of the times to the index, there should be far fewer
unique instances of the remaining metadata in many/most cases.

Change the index and index-related code to handle all time
values as integer nanoseconds since the epoch, excepting the packed
format, which (as mentioned above) is now an xstat timespec.

After these changes the index-related in-memory and on-disk time value
handling should match that of the metadata code (i.e. metadata.py,
.bupm, etc.).

Signed-off-by: Rob Browning <rlb@defaultvalue.org>
Reviewed-by: Zoran Zaric <zz@zoranzaric.de>
[rlb@defaultvalue.org: change 10e8 to 10**9 for ns per second.]
  • Loading branch information...
1 parent 6ddad71 commit 3b6f285135d5e0320b64f00ded38e293f4c5255c @rlbdv rlbdv committed Nov 10, 2012
Showing with 43 additions and 25 deletions.
  1. +3 −2 cmd/index-cmd.py
  2. +40 −23 lib/bup/index.py
View
@@ -49,11 +49,12 @@ def check_index(reader):
def update_index(top, excluded_paths):
- tmax = time.time() - 1
+ # tmax and start must be epoch nanoseconds.
+ tmax = (time.time() - 1) * 10**9
ri = index.Reader(indexfile)
wi = index.Writer(indexfile, tmax)
rig = IterHelper(ri.iter(name=top))
- tstart = int(time.time())
+ tstart = int(time.time()) * 10**9
hlinks = hlinkdb.HLinkDB(indexfile + '.hlink')
View
@@ -4,10 +4,17 @@
EMPTY_SHA = '\0'*20
FAKE_SHA = '\x01'*20
-INDEX_HDR = 'BUPI\0\0\0\3'
-# Use 64-bit integers for mtime/ctime to handle NTFS zero (Y1600) and Y2038.
-INDEX_SIG = '!QQQqqIIQII20sHII'
+INDEX_HDR = 'BUPI\0\0\0\4'
+
+# Time values are handled as integer nanoseconds since the epoch in
+# memory, but are written as xstat/metadata timespecs. This behavior
+# matches the existing metadata/xstat/.bupm code.
+
+# Record times (mtime, ctime, atime) as xstat/metadata timespecs, and
+# store all of the times in the index so they won't interfere with the
+# forthcoming metadata cache.
+INDEX_SIG = '!QQQqQqQqQIIQII20sHII'
ENTLEN = struct.calcsize(INDEX_SIG)
FOOTER_SIG = '!Q'
@@ -75,42 +82,48 @@ def __init__(self, basename, name, tmax):
self.children_n = 0
def __repr__(self):
- return ("(%s,0x%04x,%d,%d,%d,%d,%d,%d,%d,%s/%s,0x%04x,0x%08x/%d)"
+ return ("(%s,0x%04x,%d,%d,%d,%d,%d,%d,%d,%d,%s/%s,0x%04x,0x%08x/%d)"
% (self.name, self.dev, self.ino, self.nlink,
- self.ctime, self.mtime, self.uid, self.gid,
+ self.ctime, self.mtime, self.atime, self.uid, self.gid,
self.size, self.mode, self.gitmode,
self.flags, self.children_ofs, self.children_n))
def packed(self):
try:
+ ctime = xstat.nsecs_to_timespec(self.ctime)
+ mtime = xstat.nsecs_to_timespec(self.mtime)
+ atime = xstat.nsecs_to_timespec(self.atime)
return struct.pack(INDEX_SIG,
- self.dev, self.ino, self.nlink,
- self.ctime, self.mtime,
- self.uid, self.gid, self.size, self.mode,
- self.gitmode, self.sha, self.flags,
- self.children_ofs, self.children_n)
+ self.dev, self.ino, self.nlink,
+ ctime[0], ctime[1],
+ mtime[0], mtime[1],
+ atime[0], atime[1],
+ self.uid, self.gid, self.size, self.mode,
+ self.gitmode, self.sha, self.flags,
+ self.children_ofs, self.children_n)
except (DeprecationWarning, struct.error), e:
log('pack error: %s (%r)\n' % (e, self))
raise
def from_stat(self, st, tstart):
old = (self.dev, self.ino, self.nlink, self.ctime, self.mtime,
self.uid, self.gid, self.size, self.flags & IX_EXISTS)
- new = (st.st_dev, st.st_ino, st.st_nlink,
- xstat.fstime_floor_secs(st.st_ctime),
- xstat.fstime_floor_secs(st.st_mtime),
+ new = (st.st_dev, st.st_ino, st.st_nlink, st.st_ctime, st.st_mtime,
st.st_uid, st.st_gid, st.st_size, IX_EXISTS)
self.dev = st.st_dev
self.ino = st.st_ino
self.nlink = st.st_nlink
- self.ctime = xstat.fstime_floor_secs(st.st_ctime)
- self.mtime = xstat.fstime_floor_secs(st.st_mtime)
+ self.ctime = st.st_ctime
+ self.mtime = st.st_mtime
+ self.atime = st.st_atime
self.uid = st.st_uid
self.gid = st.st_gid
self.size = st.st_size
self.mode = st.st_mode
self.flags |= IX_EXISTS
- if xstat.fstime_floor_secs(st.st_ctime) >= tstart or old != new \
+ # Check that the ctime's "second" is at or after tstart's.
+ ctime_sec_in_ns = xstat.fstime_floor_secs(st.st_ctime) * 10**9
+ if ctime_sec_in_ns >= tstart or old != new \
or self.sha == EMPTY_SHA or not self.gitmode:
self.invalidate()
self._fixup()
@@ -175,22 +188,23 @@ def write(self, f):
class NewEntry(Entry):
- def __init__(self, basename, name, tmax, dev, ino, nlink, ctime, mtime,
+ def __init__(self, basename, name, tmax, dev, ino, nlink,
+ ctime, mtime, atime,
uid, gid, size, mode, gitmode, sha, flags,
children_ofs, children_n):
Entry.__init__(self, basename, name, tmax)
- (self.dev, self.ino, self.nlink, self.ctime, self.mtime,
+ (self.dev, self.ino, self.nlink, self.ctime, self.mtime, self.atime,
self.uid, self.gid, self.size, self.mode, self.gitmode, self.sha,
self.flags, self.children_ofs, self.children_n
- ) = (dev, ino, nlink, int(ctime), int(mtime), uid, gid,
+ ) = (dev, ino, nlink, ctime, mtime, atime, uid, gid,
size, mode, gitmode, sha, flags, children_ofs, children_n)
self._fixup()
class BlankNewEntry(NewEntry):
def __init__(self, basename, tmax):
NewEntry.__init__(self, basename, basename, tmax,
- 0, 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, EMPTY_SHA, 0, 0, 0)
@@ -200,10 +214,14 @@ def __init__(self, parent, basename, name, m, ofs):
self.parent = parent
self._m = m
self._ofs = ofs
- (self.dev, self.ino, self.nlink, self.ctime, self.mtime,
+ (self.dev, self.ino, self.nlink,
+ self.ctime, ctime_ns, self.mtime, mtime_ns, self.atime, atime_ns,
self.uid, self.gid, self.size, self.mode, self.gitmode, self.sha,
self.flags, self.children_ofs, self.children_n
) = struct.unpack(INDEX_SIG, str(buffer(m, ofs, ENTLEN)))
+ self.atime = xstat.timespec_to_nsecs((self.atime, atime_ns))
+ self.mtime = xstat.timespec_to_nsecs((self.mtime, mtime_ns))
+ self.ctime = xstat.timespec_to_nsecs((self.ctime, ctime_ns))
# effectively, we don't bother messing with IX_SHAMISSING if
# not IX_HASHVALID, since it's redundant, and repacking is more
@@ -416,8 +434,7 @@ def add(self, name, st, hashgen = None):
assert(isdir == endswith)
e = NewEntry(basename, name, self.tmax,
st.st_dev, st.st_ino, st.st_nlink,
- xstat.fstime_floor_secs(st.st_ctime),
- xstat.fstime_floor_secs(st.st_mtime),
+ st.st_ctime, st.st_mtime, st.st_atime,
st.st_uid, st.st_gid,
st.st_size, st.st_mode, gitmode, sha, flags,
0, 0)

0 comments on commit 3b6f285

Please sign in to comment.