Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Refactored much code out into abstract base classes

Staleness tracking is now optional - this improves BEXML's parse speed to parity with BE's (without id cache)

Parser now resets issue and comment objects after yield, thus throwing away any loaded issue. This should help prevent memory overload denial of service attacks.
  • Loading branch information...
commit 8b19a232cd9eb2a334da8c3518363a9c1befd139 1 parent 418f4d8
Niall Douglas (s [underscore] sourceforge {at} nedprod [dot] com) authored
View
64 BEXML/libBEXML/comment.py
@@ -37,6 +37,7 @@
from propertieddictionary import PropertiedDictionary
from coerce_datetime import coerce_datetime
+from abc import ABCMeta, abstractmethod, abstractproperty
from uuid import UUID
from datetime import datetime
import mimetypes
@@ -48,6 +49,7 @@
class Comment(PropertiedDictionary):
"""Base class for comments"""
+ __metaclass__=ABCMeta
mime_types=mimetypes.types_map.values()
mime_types+=mimetypes.common_types.values()
mime_types.sort()
@@ -58,7 +60,9 @@ class Comment(PropertiedDictionary):
def __init__(self, parentIssue, *entries, **args):
PropertiedDictionary.__init__(self)
self.__parent=parentIssue
- self._dirty=False
+ self.__loaded=False
+ self.__dirty=False
+ self.__trackStaleness=False
self._addProperty("uuid", "The uuid of the comment", lambda x: x if isinstance(x, UUID) else UUID(x), self.nullUUID)
self._addProperty("alt-id", "The alt-id of the comment", str, "")
self._addProperty("short-name", "The short name of the comment", str, "")
@@ -81,9 +85,33 @@ def parentIssue(self):
return self.__parent
@property
+ def isLoaded(self):
+ """True if comment has been loaded from backing store"""
+ return self.__loaded
+ @isLoaded.setter
+ def isLoaded(self, value):
+ self.__loaded=value
+
+ @property
def isDirty(self):
"""True if this comment has been modified and needs writing to disk"""
- return self._dirty
+ return self.__dirty
+ @isDirty.setter
+ def isDirty(self, value):
+ self.__dirty=value
+
+ @property
+ def tracksStaleness(self):
+ """True is staleness is tracked"""
+ return self.__trackStaleness
+ @tracksStaleness.setter
+ def tracksStaleness(self, value):
+ self.__trackStaleness=value
+
+ @abstractproperty
+ def isStale(self):
+ """True if the file backing for this comment is newer than us"""
+ pass
def match(self, commentfilter):
"""Returns true if this comment matches commentfilter"""
@@ -105,6 +133,38 @@ def match(self, commentfilter):
if not re.search(commentfilter.body, self.body): return False
return True
+ @abstractmethod
+ def load(self, reload=False):
+ """Loads in the comment from the backing store"""
+ pass
+
+ def __getitem__(self, name):
+ if self._isProperty(name) and not self.isLoaded and name is not 'uuid':
+ self.load()
+ return PropertiedDictionary.__getitem__(self, name)
+ def __getattr__(self, name):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__getattr__(self, name)
+
+ def __setitem__(self, name, value):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__setitem__(self, name, value)
+ def __setattr__(self, name, value):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__setattr__(self, name, value)
+
+ def __delitem__(self, name):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__delitem__(self, name)
+ def __delattr__(self, name):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__delattr__(self, name)
+
if __name__=="__main__":
import doctest
doctest.testmod()
View
71 BEXML/libBEXML/issue.py
@@ -34,6 +34,7 @@
</bug>
"""
+from abc import ABCMeta, abstractmethod, abstractproperty
import re
from uuid import UUID
from datetime import datetime
@@ -43,7 +44,8 @@
from coerce_datetime import coerce_datetime
class Issue(PropertiedDictionary):
- """Base class for issues"""
+ """Abstract base class for issues"""
+ __metaclass__=ABCMeta
severities=["target",
"wishlist",
"minor",
@@ -62,7 +64,9 @@ class Issue(PropertiedDictionary):
def __init__(self, *entries, **args):
PropertiedDictionary.__init__(self)
+ self.__loaded=False
self.__dirty=False
+ self.__trackStaleness=False
self._addProperty("uuid", "The uuid of the issue", lambda x: x if isinstance(x, UUID) else UUID(x), self.nullUUID)
self._addProperty("short-name", "The short name of the issue", str, "")
self._addProperty("severity", "The severity of the issue", self.__coerce_severity, "")
@@ -87,9 +91,33 @@ def __coerce_status(self, v):
return v
@property
- def isDirty(self, includeChildren=False):
+ def isLoaded(self):
+ """True if issue has been loaded from filing system"""
+ return self.__loaded
+ @isLoaded.setter
+ def isLoaded(self, value):
+ self.__loaded=value
+
+ @property
+ def isDirty(self):
"""True if this comment has been modified and needs writing to disk"""
return self.__dirty
+ @isDirty.setter
+ def isDirty(self, value):
+ self.__dirty=value
+
+ @property
+ def tracksStaleness(self):
+ """True is staleness is tracked"""
+ return self.__trackStaleness
+ @tracksStaleness.setter
+ def tracksStaleness(self, value):
+ self.__trackStaleness=value
+
+ @abstractproperty
+ def isStale(self):
+ """True if the backing for this issue is newer than us"""
+ pass
def match(self, issuefilter):
"""Returns true if this issue matches issuefilter"""
@@ -113,6 +141,7 @@ def match(self, issuefilter):
def addComment(self, comment):
"""Adds a comment to the issue"""
+ assert isinstance(comment, Comment)
self.comments[comment.uuid]=comment
return comment
@@ -127,6 +156,44 @@ def removeComment(self, comment):
else:
raise LookupError, "comment is not a string, uuid or comment"
+ @abstractmethod
+ def load(self, reload=False):
+ """Loads in the issue from the backing store"""
+ pass
+
+ @abstractmethod
+ def unload(self):
+ """Releases any data used by this issue"""
+ pass
+
+ def __getitem__(self, name):
+ if self._isProperty(name) and not self.isLoaded and name is not 'uuid':
+ self.load()
+ return PropertiedDictionary.__getitem__(self, name)
+ def __getattr__(self, name):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__getattr__(self, name)
+
+ def __setitem__(self, name, value):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__setitem__(self, name, value)
+ def __setattr__(self, name, value):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__setattr__(self, name, value)
+
+ def __delitem__(self, name):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__delitem__(self, name)
+ def __delattr__(self, name):
+ if self._isProperty(name) and not self.isLoaded:
+ self.load()
+ return PropertiedDictionary.__delattr__(self, name)
+
+
if __name__=="__main__":
import doctest
doctest.testmod()
View
146 BEXML/libBEXML/parsers/be_dir.py
@@ -21,19 +21,11 @@ def __init__(self, parentIssue, dirpath, encoding):
CommentBase.__init__(self, parentIssue)
self.dirpath=dirpath
self.encoding=encoding
- self.__loaded=True
+ self.isLoaded=True
self.uuid=os.path.basename(dirpath)
- self.__loaded=False
+ self.isLoaded=False
self.stat=None
- @property
- def isLoaded(self):
- """True if comment has been loaded from filing system"""
- return self.__loaded
- @isLoaded.setter
- def isLoaded(self, value):
- self.__loaded=value
-
def __getStat(self):
"""Returns the stat for the file backing of this comment"""
stat=namedtuple('CommentStat', ['values', 'body'])
@@ -44,14 +36,14 @@ def __getStat(self):
@property
def isStale(self):
"""True if the file backing for this comment is newer than us"""
- if not self.__loaded:
+ if not self.isLoaded or not self.tracksStaleness:
return None
stat=self.__getStat()
return stat.values!=self.stat.values
def load(self, reload=False):
"""Loads in the comment from the filing system"""
- if not reload and self.__loaded:
+ if not reload and self.isLoaded:
return
with codecs.open(os.path.join(self.dirpath, "values"), 'r', self.encoding) as ih:
values=yaml.safe_load(ih)
@@ -61,36 +53,11 @@ def load(self, reload=False):
notloaded=self._load_mostly(values)
if len(notloaded)>0:
log.warn("The following values from comment '"+self.dirpath+"' were not recognised: "+repr(notloaded))
- self.__loaded=True
- self._dirty=False
- self.stat=self.__getStat()
-
- def __getitem__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return CommentBase.__getitem__(self, name)
- def __getattr__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return CommentBase.__getattr__(self, name)
-
- def __setitem__(self, name, value):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return CommentBase.__setitem__(self, name, value)
- def __setattr__(self, name, value):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return CommentBase.__setattr__(self, name, value)
-
- def __delitem__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return CommentBase.__delitem__(self, name)
- def __delattr__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return CommentBase.__delattr__(self, name)
+ self.isLoaded=True
+ self.isDirty=False
+ if self.tracksStaleness:
+ self.stat=self.__getStat()
+
class BEDirIssue(IssueBase):
"""An issue loaded from a filing system based BE repo"""
@@ -99,16 +66,11 @@ def __init__(self, dirpath, encoding):
IssueBase.__init__(self)
self.dirpath=dirpath
self.encoding=encoding
- self.__loaded=True
+ self.isLoaded=True
self.uuid=os.path.basename(dirpath)
- self.__loaded=False
+ self.isLoaded=False
self.stat=None
- @property
- def isLoaded(self):
- """True if issue has been loaded from filing system"""
- return self.__loaded
-
def __getStat(self):
"""Returns the stat for the file backing of this issue"""
stat=namedtuple('IssueStat', ['values'])
@@ -118,34 +80,34 @@ def __getStat(self):
@property
def isStale(self):
"""True if the file backing for this issue is newer than us"""
- if not self.__loaded:
+ if not self.isLoaded or not self.tracksStaleness:
return None
stat=self.__getStat()
return stat.values!=self.stat.values
def addComment(self, dirpath):
"""Adds a comment to the issue"""
- temp1=self.__loaded
+ temp1=self.isLoaded
comment=BEDirComment(self, dirpath, self.encoding)
try:
- self.__loaded=True
+ self.isLoaded=True
comment.isLoaded=True
return IssueBase.addComment(self, comment)
finally:
comment.isLoaded=False
- self.__loaded=temp1
+ self.isLoaded=temp1
def removeComment(self, comment):
- temp=self.__loaded
+ temp=self.isLoaded
try:
- self.__loaded=True
+ self.isLoaded=True
return IssueBase.removeComment(self, comment)
finally:
- self.__loaded=temp
+ self.isLoaded=temp
def load(self, reload=False):
"""Loads in the issue from the filing system"""
- if not reload and self.__loaded:
+ if not reload and self.isLoaded:
return
with codecs.open(os.path.join(self.dirpath, "values"), 'r', self.encoding) as ih:
values=yaml.safe_load(ih)
@@ -153,49 +115,23 @@ def load(self, reload=False):
notloaded=self._load_mostly(values)
if len(notloaded)>0:
log.warn("The following values from issue '"+self.dirpath+"' were not recognised: "+repr(notloaded))
- self.__loaded=True
- self._dirty=False
- self.stat=self.__getStat()
-
- def __getitem__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return IssueBase.__getitem__(self, name)
- def __getattr__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return IssueBase.__getattr__(self, name)
-
- def __setitem__(self, name, value):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return IssueBase.__setitem__(self, name, value)
- def __setattr__(self, name, value):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return IssueBase.__setattr__(self, name, value)
-
- def __delitem__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return IssueBase.__delitem__(self, name)
- def __delattr__(self, name):
- if self._isProperty(name) and not self.isLoaded:
- self.load()
- return IssueBase.__delattr__(self, name)
+ self.isLoaded=True
+ self.isDirty=False
+ if self.tracksStaleness:
+ self.stat=self.__getStat()
class BEDirParser(ParserBase):
"""Parses a filing system based BE repo"""
uuid_match=re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")
- def __init__(self, uri, encoding="UTF-8", cache_in_memory=True):
+ def __init__(self, uri, encoding="UTF-8", cache_in_memory=False, precache_in_memory=False):
ParserBase.__init__(self, uri)
self.version="" # This repo's version string
self.encoding=encoding # How to treat text files in this repo
+ self.cache_in_memory=cache_in_memory or precache_in_memory
+ self.precache_in_memory=precache_in_memory
self.__bedir={} # Dictionary of bug directories in repo
- if not cache_in_memory:
- log.warn("cache_in_memory=False not supported and therefore ignored")
def try_location(self):
path=self._pathFromURI()
@@ -215,6 +151,16 @@ def try_location(self):
return (-996, "Can't find a uuid bug directory")
return (1, None)
+ def __loadIssueAndComments(self, bug, bugspath):
+ """Adds an unloaded issue and comments"""
+ self.__bedir[bugspath][bug]=bugitem=BEDirIssue(os.path.join(bugspath, bug), self.encoding)
+ commentspath=os.path.join(bugspath, bug, "comments")
+ if os.path.exists(commentspath):
+ comments=filter(self.uuid_match.match, os.listdir(commentspath))
+ for comment in comments:
+ bugitem.addComment(os.path.join(commentspath, comment))
+ return bugitem
+
def reload(self):
"""Loads a BE directory structure into memory"""
if self.version=="":
@@ -225,15 +171,14 @@ def reload(self):
for item in items:
bugspath=os.path.join(path, item, "bugs")
if os.path.exists(bugspath):
- self.__bedir[bugspath]=bugsdict={}
+ self.__bedir[bugspath]={}
bugs=filter(self.uuid_match.match, os.listdir(bugspath))
for bug in bugs:
- bugsdict[bug]=bugitem=BEDirIssue(os.path.join(bugspath, bug), self.encoding)
- commentspath=os.path.join(bugspath, bug, "comments")
- if os.path.exists(commentspath):
- comments=filter(self.uuid_match.match, os.listdir(commentspath))
- for comment in comments:
- bugitem.addComment(os.path.join(commentspath, comment))
+ self.__loadIssueAndComments(bug, bugspath)
+ if self.precache_in_memory:
+ for issue in self.parse():
+ for commentuuid in issue.comments:
+ issue.comments[commentuuid].uuid
def parse(self, issuefilter=None):
if len(self.__bedir)==0:
@@ -243,13 +188,18 @@ def parse(self, issuefilter=None):
for issueuuid in issueuuids:
issue=issueuuids[issueuuid]
# Refresh if loaded and stale
- if issue.isLoaded and issue.isStale:
+ if issue.isLoaded and issue.tracksStaleness and issue.isStale:
issue.load(True)
if issuefilter is None:
yield issue
else:
if issuefilter.match(issue):
yield issue
+ if not self.cache_in_memory:
+ # Replace with a fresh structure. If the caller took
+ # a copy of the issue, it'll live on, otherwise it'll
+ # get GCed
+ self.__loadIssueAndComments(issueuuid, os.path.dirname(issue.dirpath))
def instantiate(uri, **args):
return BEDirParser(uri, **args)
View
8 BEXML/tests/TestParseBErepoWithLib.py
@@ -25,17 +25,17 @@ def test(self):
start=time.time()
for issue in parser.parse():
+ issue.status
for commentuuid in issue.comments:
- issue.comments[commentuuid].uuid
- pass
+ issue.comments[commentuuid].alt_id
end=time.time()
print("Reading the bugs everywhere repository for the first time took %f secs" % (end-start-self.emptyloop))
start=time.time()
for issue in parser.parse():
+ issue.status
for commentuuid in issue.comments:
- issue.comments[commentuuid].uuid
- pass
+ issue.comments[commentuuid].alt_id
end=time.time()
print("Reading the bugs everywhere repository for the second time took %f secs" % (end-start-self.emptyloop))
#for issue in parser.parse():
Please sign in to comment.
Something went wrong with that request. Please try again.