Browse files

distribute - add LazySeqArtifact to allow lazy loading of sequence ar…


Ignore-this: 2290383ea30891a379221e312a3fc7c4

  • Loading branch information...
MattShannon committed Jul 15, 2013
1 parent b11aacb commit 56d422316ea7f8a0a931d49462fe788069f27bf4
Showing with 40 additions and 0 deletions.
  1. +40 −0 armspeech/bisque/
@@ -243,6 +243,46 @@ def loc(self, buildRepo):
def isDone(self, buildRepo):
return os.path.exists(self.location)
+class LazySeqArtifact(Artifact):
+ """A wrapper for sequence-valued artifacts allowing lazy loading.
+ This (slightly hacky) class provides a wrapper around an underlying
+ sequence-valued artifact.
+ The artifact value for this class is an iterator representing the same
+ sequence as the underlying value.
+ When loadValue is called for this artifact an iterator is returned, but the
+ underlying value is only computed when the first element of the iterator is
+ accessed.
+ The underlying value is then stored in memory to allow it to be used for
+ the remainder of the lifetime of the iterator.
+ This class therefore provides a way for a given sequence-valued artifact to
+ be loaded (e.g. unpickled) only when it is needed.
+ """
+ def __init__(self, seqArt):
+ self.seqArt = seqArt
+ def parents(self):
+ return self.seqArt.parents()
+ def parentArtifacts(self):
+ return self.seqArt.parentArtifacts()
+ def computeSecHash(self):
+ return self.seqArt.computeSecHash()
+ def loc(self, buildRepo):
+ return self.seqArt.loc(buildRepo)
+ def isDone(self, buildRepo):
+ return self.seqArt.isDone(buildRepo)
+ def loadValue(self, buildRepo):
+ def getValueNew():
+ for elem in self.seqArt.loadValue(buildRepo):
+ yield elem
+ return getValueNew()
+ def saveValue(self, buildRepo, value):
+ return self.seqArt.saveValue(buildRepo, value)
+def lazySeq(seqArt):
+ return LazySeqArtifact(seqArt)
@codeDeps(Artifact, codedep.getHash, persist.loadPickle, persist.savePickle,

0 comments on commit 56d4223

Please sign in to comment.