Skip to content

Commit

Permalink
Avoid full prov (#37)
Browse files Browse the repository at this point in the history
* avoid keeping full provenance in form of nested dependencies

* assemble full readonly list from dependencies in ctor
* before we would have kept fully connected upstream provenance
  for each state which can become large quick

* sorting
  • Loading branch information
lukasheinrich committed Mar 27, 2018
1 parent cb9471c commit 4cd39e9
Showing 1 changed file with 13 additions and 23 deletions.
36 changes: 13 additions & 23 deletions packtivity/statecontexts/posixfs_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,23 +18,20 @@ def __init__(self,readwrite = None, readonly = None, dependencies = None, identi
except AssertionError:
raise TypeError('readwrite and readonly must be None or a list {} {}'.format(type(readonly)))
self._identifier = identifier
self.readwrite = list(map(os.path.realpath,readwrite) if readwrite else [])
self.add_readonly = list(map(os.path.realpath,readonly) if readonly else [])
self.dependencies = dependencies or []
self.datamodel = None

def __repr__(self):
return '<LocalFSState rw: {}, ro: {}>'.format(self.readwrite,self.readonly)

@property
def readonly(self):
readonlies = [x for x in self.add_readonly]
for d in self.dependencies or []:
readonlies = list(map(os.path.realpath,readonly) if readonly else [])
for d in dependencies or []:
if d.readwrite:
readonlies += d.readwrite # if dep has readwrite add those
else:
readonlies += d.readonly # else add the readonlies
return list(map(os.path.realpath,readonlies))

self.readwrite = sorted(list(map(os.path.realpath,readwrite) if readwrite else []))
self.readonly = sorted(list(map(os.path.realpath,readonlies)))
self.datamodel = None

def __repr__(self):
return '<LocalFSState rw: {}, ro: {}>'.format(self.readwrite,self.readonly)


@property
Expand All @@ -46,9 +43,6 @@ def metadir(self):
def identifier(self):
return self._identifier

def add_dependency(self,depstate):
self.dependencies.append(depstate)

def reset(self):
'''
resets state by deleting readwrite directory contents (deletes tree and re-creates)
Expand All @@ -72,10 +66,8 @@ def state_hash(self):
checks both readwrite directories and dependencies (assumed to be subtrees of readwrite directories)
return: SHA1 hash
'''

#hash the upstream / input state
depwrites = [deprw for dep in self.dependencies for deprw in dep.readwrite]
dep_checksums = [checksumdir.dirhash(d) for d in depwrites if os.path.isdir(d)]
dep_checksums = [checksumdir.dirhash(d) for d in self.readonly if os.path.isdir(d)]

#hash out writing state
state_checksums = [checksumdir.dirhash(d) for d in self.readwrite if os.path.isdir(d)]
Expand Down Expand Up @@ -105,15 +97,13 @@ def json(self):
'state_type': 'localfs',
'identifier': self.identifier(),
'readwrite': self.readwrite,
'add_readonly': self.add_readonly,
'dependencies': [x.json() for x in self.dependencies]
'readonly': self.readonly,
}

@classmethod
def fromJSON(cls,jsondata):
return cls(
readwrite = jsondata['readwrite'],
readonly = jsondata['add_readonly'],
identifier = jsondata['identifier'],
dependencies = [LocalFSState.fromJSON(x) for x in jsondata['dependencies']]
readwrite = sorted(jsondata['readwrite']),
readonly = sorted(jsondata['readonly']),
)

0 comments on commit 4cd39e9

Please sign in to comment.