@@ -123,7 +123,7 @@ Pootle: for dirpath, subdirs, filenames in os.walk(self.po_dir, topdown=False): for name in filenames: pofilename = os.path.join(dirpath, name) - parsedfile = pootlefile.pootlefile(pofilename=pofilename, generatestats=True) + parsedfile = pootlefile.pootlefile(pofilename=pofilename) count += len(parsedfile.units) print "stats on %d units" % count @@ -167,9 +167,9 @@ Pootle: args = {"pofilename": pofilename, "submit0": "true", "trans0": "changed"} page = self.server.getpage(["zxx", "benchmark", "translate.html"], session, args) pofile = project.getpofile(pofilename) - print str(pofile.transunits[0]) + print str(pofile.getitem(0)) # assert fails because of multistring - # assert pofile.transunits[0].unquotedmsgstr == "changed" + # assert pofile.getitem(0).unquotedmsgstr == "changed" print page.templatevars if __name__ == "__main__": benchmark.py @@ -32,7 +32,8 @@ def processfile(filename): pofile = pootlefile.pootlefile(dummyproject, os.path.basename(filename)) pofile.readpofile() conflictitems = [] - for item, poentry in enumerate(pofile.transunits): + for item in pofile.statistics.getstats()["total"]: + poentry = pofile.units[item] if poentry.hasplural(): targets = poentry.target.strings else: conflict2suggest.py @@ -332,21 +332,19 @@ class ProjectIndex(pagelayout.PootleNavPage): mainstats = "" mainicon = "file" else: - if dirfilter or self.editing or self.showassigns or self.showchecks: + pofilenames = self.project.browsefiles(dirfilter) + projecttotals = self.project.getquickstats(pofilenames) + if self.editing or self.showassigns or self.showchecks: # we need the complete stats - pofilenames = self.project.browsefiles(dirfilter) projectstats = self.project.combinestats(pofilenames) else: - # a common case: plain stats table we can take a shortcut - pofilenames = self.project.browsefiles() - projectstats = self.project.getquickstats() + projectstats = projecttotals if self.editing: actionlinks = self.getactionlinks("", projectstats, ["editing", "mine", "review", "check", "assign", "goal", "quick", "all", "zip", "sdf"], dirfilter) else: actionlinks = self.getactionlinks("", projectstats, ["editing", "goal", "zip", "sdf"]) - mainstats = self.getitemstats("", projectstats, len(pofilenames)) - maindata = self.getstats(self.project, projectstats) - mainicon = "folder" + mainstats = self.getitemstats("", pofilenames, len(pofilenames)) + mainstats["summary"] = self.describestats(self.project, projecttotals, len(pofilenames)) if self.showgoals: childitems = self.getgoalitems(dirfilter) else: @@ -490,11 +488,11 @@ class ProjectIndex(pagelayout.PootleNavPage): if assignwhich == "all": pass elif assignwhich == "untranslated": - search.matchnames = ["fuzzy", "blank"] + search.matchnames = ["fuzzy", "untranslated"] elif assignwhich == "unassigned": search.assignedto = [None] elif assignwhich == "unassigneduntranslated": - search.matchnames = ["fuzzy", "blank"] + search.matchnames = ["fuzzy", "untranslated"] search.assignedto = [None] else: raise ValueError("unexpected assignwhich") @@ -683,7 +681,8 @@ class ProjectIndex(pagelayout.PootleNavPage): goal["goal"]["show_adduser"] = True goal["goal"]["otherusers"] = unassignedusers goal["goal"]["adduser_title"] = self.localize("Add User") - goal["stats"] = self.getitemstats("", projectstats, len(pofilenames)) + goal["stats"] = self.getitemstats("", pofilenames, len(pofilenames)) + projectstats = self.project.getquickstats(pofilenames) goal["data"] = self.getstats(self.project, projectstats) return goal @@ -694,7 +693,7 @@ class ProjectIndex(pagelayout.PootleNavPage): goalfilenames = self.project.getgoalfiles(self.currentgoal, dirfilter=direntry, includedirs=False, expanddirs=True) projectstats = self.project.combinestats(goalfilenames) else: - projectstats = self.project.combinestats(pofilenames) + projectstats = self.project.combine_totals(pofilenames) basename = os.path.basename(direntry) browseurl = self.getbrowseurl("%s/" % basename, **newargs) diritem = {"href": browseurl, "title": basename, "icon": "folder", "isdir": True} @@ -702,10 +701,12 @@ class ProjectIndex(pagelayout.PootleNavPage): actionlinks = self.getactionlinks(basename, projectstats, linksrequired=linksrequired) diritem["actions"] = actionlinks if self.showgoals and "goal" in self.argdict: - diritem["stats"] = self.getitemstats(basename, projectstats, (len(goalfilenames), len(pofilenames))) + diritem["stats"] = self.getitemstats(basename, goalfilenames, (len(goalfilenames), len(pofilenames))) + projectstats = self.project.getquickstats(goalfilenames) diritem["data"] = self.getstats(self.projects, projectstats) else: - diritem["stats"] = self.getitemstats(basename, projectstats, len(pofilenames)) + diritem["stats"] = self.getitemstats(basename, pofilenames, len(pofilenames)) + projectstats = self.project.getquickstats(pofilenames) diritem["data"] = self.getstats(self.project, projectstats) return diritem @@ -717,7 +718,7 @@ class ProjectIndex(pagelayout.PootleNavPage): else: linksrequired = ["mine", "review", "quick", "all", "po", "xliff", "update", "commit"] basename = os.path.basename(fileentry) - projectstats = self.project.combinestats([fileentry]) + projectstats = self.project.combine_totals([fileentry]) browseurl = self.getbrowseurl(basename, **newargs) fileitem = {"href": browseurl, "title": basename, "icon": "file", "isfile": True} actions = self.getactionlinks(basename, projectstats, linksrequired=linksrequired) @@ -762,7 +763,8 @@ class ProjectIndex(pagelayout.PootleNavPage): else: actionlink["sep"] = "" fileitem["actions"] = actions - fileitem["stats"] = self.getitemstats(basename, projectstats, None) + fileitem["stats"] = self.getitemstats(basename, [fileentry], None) + projectstats = self.project.getquickstats([fileentry]) fileitem["data"] = self.getstats(self.project, projectstats) return fileitem @@ -786,7 +788,7 @@ class ProjectIndex(pagelayout.PootleNavPage): else: action = None assignstats = self.project.combineassignstats(assignfilenames, action) - assignusers = [username.replace("assign-", "", 1) for username in assignstats.iterkeys()] + assignusers = list(assignstats.iterkeys()) useroptions += [username for username in assignusers if username not in useroptions] if len(assignusers) > 1: multiusers = "multiple" @@ -859,34 +861,34 @@ class ProjectIndex(pagelayout.PootleNavPage): minelink = self.localize("Translate My Strings") else: minelink = self.localize("View My Strings") - mystats = projectstats.get("assign-%s" % self.session.username, []) + mystats = projectstats.get('assign', {}).get(self.session.username, []) if len(mystats): minelink = {"href": self.makelink(baseactionlink, assignedto=self.session.username), "text": minelink} else: minelink = {"title": self.localize("No strings assigned to you"), "text": minelink} actionlinks.append(minelink) if "quick" in linksrequired and "translate" in self.rights: - mytranslatedstats = [statsitem for statsitem in mystats if statsitem in projectstats.get("translated", [])] + mytranslatedstats = [statsitem for statsitem in mystats if statsitem in projectstats["units"].get("translated", [])] quickminelink = self.localize("Quick Translate My Strings") if len(mytranslatedstats) < len(mystats): - quickminelink = {"href": self.makelink(baseactionlink, assignedto=self.session.username, fuzzy=1, blank=1), "text": quickminelink} + quickminelink = {"href": self.makelink(baseactionlink, assignedto=self.session.username, fuzzy=1, untranslated=1), "text": quickminelink} else: quickminelink = {"title": self.localize("No untranslated strings assigned to you"), "text": quickminelink} actionlinks.append(quickminelink) - if "review" in linksrequired and projectstats.get("has-suggestion", []): + if "review" in linksrequired and projectstats.get("units", {}).get("check-hassuggestion", []): if "review" in self.rights: reviewlink = self.localize("Review Suggestions") else: reviewlink = self.localize("View Suggestions") - reviewlink = {"href": self.makelink(baseactionlink, review=1, **{"has-suggestion": 1}), "text": reviewlink} + reviewlink = {"href": self.makelink(baseactionlink, review=1, **{"hassuggestion": 1}), "text": reviewlink} actionlinks.append(reviewlink) if "quick" in linksrequired: if "translate" in self.rights: quicklink = self.localize("Quick Translate") else: quicklink = self.localize("View Untranslated") - if len(projectstats.get("translated", [])) < len(projectstats.get("total", [])): - quicklink = {"href": self.makelink(baseactionlink, fuzzy=1, blank=1), "text": quicklink} + if projectstats.get("translated", 0) < projectstats.get("total", 0): + quicklink = {"href": self.makelink(baseactionlink, fuzzy=1, untranslated=1), "text": quicklink} else: quicklink = {"title": self.localize("No untranslated items"), "text": quicklink} actionlinks.append(quicklink) @@ -928,17 +930,16 @@ class ProjectIndex(pagelayout.PootleNavPage): actions["basic"][-1]["sep"] = "" return actions - def getitemstats(self, basename, projectstats, numfiles): + def getitemstats(self, basename, pofilenames, numfiles): """returns a widget summarizing item statistics""" - statssummary = self.describestats(self.project, projectstats, numfiles) - stats = {"summary": statssummary, "checks": [], "tracks": [], "assigns": []} + stats = {"checks": [], "tracks": [], "assigns": []} if not basename or basename.endswith("/"): linkbase = basename + "translate.html?" else: linkbase = basename + "?translate=1" - if projectstats: + if pofilenames: if self.showchecks: - stats["checks"] = self.getcheckdetails(projectstats, linkbase) + stats["checks"] = self.getcheckdetails(pofilenames, linkbase) if self.showtracks: trackfilter = (self.dirfilter or "") + basename trackpofilenames = self.project.browsefiles(trackfilter) @@ -949,15 +950,16 @@ class ProjectIndex(pagelayout.PootleNavPage): removelinkbase = "?showassigns=1&removeassigns=1" else: removelinkbase = "?showassigns=1&removeassigns=1&removefilter=%s" % basename - stats["assigns"] = self.getassigndetails(projectstats, linkbase, removelinkbase) + stats["assigns"] = self.getassigndetails(pofilenames, linkbase, removelinkbase) return stats def gettrackdetails(self, projecttracks, linkbase): """return a list of strings describing the results of tracks""" return [trackmessage for trackmessage in projecttracks] - def getcheckdetails(self, projectstats, linkbase): + def getcheckdetails(self, pofilenames, linkbase): """return a list of strings describing the results of checks""" + projectstats = self.project.combine_unit_stats(pofilenames) total = max(len(projectstats.get("total", [])), 1) checklinks = [] keys = projectstats.keys() @@ -969,31 +971,28 @@ class ProjectIndex(pagelayout.PootleNavPage): checkname = checkname.replace("check-", "", 1) if total and checkcount: stats = self.nlocalize("%d string (%d%%) failed", "%d strings (%d%%) failed", checkcount, checkcount, (checkcount * 100 / total)) - checklink = {"href": self.makelink(linkbase, **{checkname:1}), "text": checkname, "stats": stats} + checklink = {"href": self.makelink(linkbase, **{str(checkname):1}), "text": checkname, "stats": stats} checklinks += [checklink] return checklinks - def getassigndetails(self, projectstats, linkbase, removelinkbase): + def getassigndetails(self, pofilenames, linkbase, removelinkbase): """return a list of strings describing the assigned strings""" # TODO: allow setting of action, so goals can only show the appropriate action assigns - total = projectstats.get("total", []) # quick lookup of what has been translated - translated = dict.fromkeys(projectstats.get("translated", [])) - totalcount = len(total) - totalwords = self.project.countwords(total) + projectstats = self.project.combinestats(pofilenames) + totalcount = projectstats.get("total", 0) + totalwords = projectstats.get("totalsourcewords", 0) + translated = projectstats['units'].get("translated", []) assignlinks = [] - keys = projectstats.keys() + keys = projectstats['assign'].keys() keys.sort() for assignname in keys: - if not assignname.startswith("assign-"): - continue - assigned = projectstats[assignname] + assigned = projectstats['assign'][assignname] assigncount = len(assigned) assignwords = self.project.countwords(assigned) complete = [statsitem for statsitem in assigned if statsitem in translated] completecount = len(complete) completewords = self.project.countwords(complete) - assignname = assignname.replace("assign-", "", 1) if totalcount and assigncount: assignlink = {"href": self.makelink(linkbase, assignedto=assignname), "text": assignname} percentassigned = assignwords * 100 / max(totalwords, 1) indexpage.py @@ -42,6 +42,7 @@ from translate.misc import optrecurse from Pootle import __version__ as pootleversion from translate import __version__ as toolkitversion from jToolkit import __version__ as jtoolkitversion +from Pootle import statistics try: from xml.etree import ElementTree except ImportError: @@ -541,6 +542,8 @@ class PootleOptionParser(simplewebserver.WebOptionParser): self.add_option('', "--refreshstats", dest="action", action="store_const", const="refreshstats", default="runwebserver", help="refresh the stats files instead of running the webserver") psycomodes=["none", "full", "profile"] + self.add_option('', "--statsdb_file", action="store", type="string", dest="statsdb_file", + default=None, help="Specifies the location of the SQLite stats db file.") try: import psyco self.add_option('', "--psyco", dest="psyco", default=None, choices=psycomodes, metavar="MODE", @@ -581,6 +584,7 @@ def main(): options, args = parser.parse_args() options.errorlevel = options.logerrors usepsyco(options) + statistics.STATS_DB_FILE = options.statsdb_file if options.action != "runwebserver": options.servertype = "dummy" server = parser.getserver(options) pootle.py @@ -26,6 +26,7 @@ from translate.storage import po from translate.storage.poheader import tzstring from translate.storage import xliff from translate.storage import factory +from translate.filters import checks from translate.misc.multistring import multistring from Pootle import __version__ from Pootle import statistics @@ -34,28 +35,7 @@ from jToolkit import glock import time import os -class Wrapper(object): - """An object which wraps an inner object, delegating to the encapsulated methods, etc""" - def __getattr__(self, attrname, *args): - if attrname in self.__dict__: - return self.__dict__[attrname] - return getattr(self.__dict__["__innerobj__"], attrname, *args) - - def __setattr__(self, attrname, value): - if attrname == "__innerobj__": - self.__dict__[attrname] = value - elif attrname in self.__dict__: - if isinstance(self.__dict__[attrname], property): - self.__dict__[attrname].fset(value) - else: - self.__dict__[attrname] = value - elif attrname in self.__class__.__dict__: - if isinstance(self.__class__.__dict__[attrname], property): - self.__class__.__dict__[attrname].fset(self, value) - else: - self.__dict__[attrname] = value - else: - return setattr(self.__dict__["__innerobj__"], attrname, value) +_UNIT_CHECKER = checks.UnitChecker() class LockedFile: """locked interaction with a filesystem file""" @@ -123,7 +103,7 @@ class pootleassigns: assignsstring = assignsfile.read() assignsfile.close() poassigns = {} - itemcount = len(getattr(self, "classify", {}).get("total", [])) + itemcount = len(getattr(self, "stats", {}).get("total", [])) for line in assignsstring.split("\n"): if not line.strip(): continue @@ -250,17 +230,16 @@ class pootleassigns: assignitems.extend(actionitems) return assignitems -class pootlefile(Wrapper): +class pootlebase(object): + pass + +class pootlefile(pootlebase): """this represents a pootle-managed file and its associated files""" - innerclass = po.pofile x_generator = "Pootle %s" % __version__.ver - def __init__(self, project=None, pofilename=None, generatestats=True): + def __init__(self, project=None, pofilename=None): if pofilename: - innerclass = factory.getclass(pofilename) - innerobj = innerclass() - self.__innerobj__ = innerobj - self.UnitClass = innerobj.UnitClass - + self.__class__.__bases__ = (factory.getclass(pofilename),) + super(pootlefile, self).__init__() self.pofilename = pofilename if project is None: from Pootle import projects @@ -275,19 +254,16 @@ class pootlefile(Wrapper): self.lockedfile = LockedFile(self.filename) # we delay parsing until it is required self.pomtime = None - self.assigns = pootleassigns(self) + self.assigns = None self.pendingfilename = self.filename + os.extsep + "pending" self.pendingfile = None - self.statistics = statistics.pootlestatistics(self, generatestats) + self.statistics = statistics.pootlestatistics(self) self.tmfilename = self.filename + os.extsep + "tm" # we delay parsing until it is required self.pomtime = None self.tracker = timecache.timecache(20*60) - def __str__(self): - return self.__innerobj__.__str__() - def parsestring(cls, storestring): newstore = cls() newstore.parse(storestring) @@ -308,39 +284,34 @@ class pootlefile(Wrapper): def getheaderplural(self): """returns values for nplural and plural values. It tries to see if the file has it specified (in a po header or similar).""" - method = getattr(self.__innerobj__, "getheaderplural", None) - if method and callable(method): - return self.__innerobj__.getheaderplural() - else: + try: + return super(pootlefile, self).getheaderplural() + except AttributeError: return None, None def updateheaderplural(self, *args, **kwargs): """updates the file header. If there is an updateheader function in the underlying store it will be delegated there.""" - method = getattr(self.__innerobj__, "updateheaderplural", None) - if method and callable(method): - self.__innerobj__.updateheaderplural(*args, **kwargs) + try: + super(pootlefile, self).updateheaderplural(*args, **kwargs) + except AttributeError: + pass def updateheader(self, **kwargs): """updates the file header. If there is an updateheader function in the underlying store it will be delegated there.""" - method = getattr(self.__innerobj__, "updateheader", None) - if method and callable(method): - self.__innerobj__.updateheader(**kwargs) + try: + super(pootlefile, self).updateheader(**kwargs) + except AttributeError: + pass def readpendingfile(self): """reads and parses the pending file corresponding to this file""" if os.path.exists(self.pendingfilename): - pendingmtime = statistics.getmodtime(self.pendingfilename) - if pendingmtime == getattr(self, "pendingmtime", None): - return inputfile = open(self.pendingfilename, "r") - self.pendingmtime, self.pendingfile = pendingmtime, factory.getobject(inputfile, ignore=".pending") - if self.pomtime: - self.reclassifysuggestions() + self.pendingfile = factory.getobject(inputfile, ignore=".pending") else: self.pendingfile = po.pofile() - self.savependingfile() def savependingfile(self): """saves changes to disk...""" @@ -348,7 +319,6 @@ class pootlefile(Wrapper): outputfile = open(self.pendingfilename, "w") outputfile.write(output) outputfile.close() - self.pendingmtime = statistics.getmodtime(self.pendingfilename) def readtmfile(self): """reads and parses the tm file corresponding to this file""" @@ -361,28 +331,9 @@ class pootlefile(Wrapper): else: self.tmfile = po.pofile() - def reclassifysuggestions(self): - """shortcut to only update classification of has-suggestion for all items""" - suggitems = [] - sugglocations = {} - for thesugg in self.pendingfile.units: - locations = tuple(thesugg.getlocations()) - sugglocations[locations] = thesugg - suggitems = [item for item in self.transunits if tuple(item.getlocations()) in sugglocations] - havesuggestions = self.statistics.classify["has-suggestion"] - for item, poel in enumerate(self.transunits): - if (poel in suggitems) != (item in havesuggestions): - if poel in suggitems: - havesuggestions.append(item) - else: - havesuggestions.remove(item) - havesuggestions.sort() - self.statistics.calcstats() - self.statistics.savestats() - def getsuggestions(self, item): """find all the suggestion items submitted for the given item""" - unit = self.transunits[item] + unit = self.getitem(item) if isinstance(unit, xliff.xliffunit): return unit.getalttrans() @@ -394,7 +345,7 @@ class pootlefile(Wrapper): def addsuggestion(self, item, suggtarget, username): """adds a new suggestion for the given item""" - unit = self.transunits[item] + unit = self.getitem(item) if isinstance(unit, xliff.xliffunit): if isinstance(suggtarget, list) and (len(suggtarget) > 0): suggtarget = suggtarget[0] @@ -415,7 +366,7 @@ class pootlefile(Wrapper): def deletesuggestion(self, item, suggitem): """removes the suggestion from the pending file""" - unit = self.transunits[item] + unit = self.getitem(item) if hasattr(unit, "xmlelement"): suggestions = self.getsuggestions(item) unit.delalttrans(suggestions[suggitem]) @@ -445,7 +396,7 @@ class pootlefile(Wrapper): def gettmsuggestions(self, item): """find all the tmsuggestion items submitted for the given item""" self.readtmfile() - unit = self.transunits[item] + unit = self.getitem(item) locations = unit.getlocations() # TODO: review the matching method # Can't simply use the location index, because we want multiple matches @@ -464,9 +415,6 @@ class pootlefile(Wrapper): pomtime, filecontents = self.lockedfile.getcontents() # note: we rely on this not resetting the filename, which we set earlier, when given a string self.parse(filecontents) - # we ignore all the headers by using this filtered set - self.transunits = [poel for poel in self.units if not (poel.isheader() or poel.isblank())] - self.statistics.classifyunits() self.pomtime = pomtime def savepofile(self): @@ -476,11 +424,17 @@ class pootlefile(Wrapper): def pofreshen(self): """makes sure we have a freshly parsed pofile""" - if not os.path.exists(self.filename): - # the file has been removed, update the project index (and fail below) - self.project.scanpofiles() - if self.pomtime != self.lockedfile.readmodtime(): - self.readpofile() + try: + if self.pomtime != self.lockedfile.readmodtime(): + self.readpofile() + except OSError, e: + # If this exception is not triggered by a bad + # symlink, then we have a missing file on our hands... + if not os.path.islink(self.filename): + # ...and thus we rescan our files to get rid of the missing filename + self.project.scanpofiles() + else: + print "%s is a broken symlink" % (self.filename,) def getoutput(self): """returns pofile output""" @@ -490,7 +444,7 @@ class pootlefile(Wrapper): def updateunit(self, item, newvalues, userprefs, languageprefs): """updates a translation with a new target value""" self.pofreshen() - unit = self.transunits[item] + unit = self.getitem(item) if newvalues.has_key("target"): unit.target = newvalues["target"] @@ -506,6 +460,12 @@ class pootlefile(Wrapper): if userprefs: if getattr(userprefs, "name", None) and getattr(userprefs, "email", None): headerupdates["Last_Translator"] = "%s <%s>" % (userprefs.name, userprefs.email) + # XXX: If we needed to add a header, the index value in item will be one out after + # adding the header. + # TODO: remove once we force the PO class to always output headers + force_recache = False + if not self.header(): + force_recache = True self.updateheader(add=True, **headerupdates) if languageprefs: nplurals = getattr(languageprefs, "nplurals", None) @@ -513,27 +473,33 @@ class pootlefile(Wrapper): if nplurals and pluralequation: self.updateheaderplural(nplurals, pluralequation) self.savepofile() + if force_recache: + self.statistics.purge_totals() self.statistics.reclassifyunit(item) + def getitem(self, item): + """Returns a single unit based on the item number.""" + return self.units[self.statistics.getstats(_UNIT_CHECKER)["total"][item]] + def iteritems(self, search, lastitem=None): """iterates through the items in this pofile starting after the given lastitem, using the given search""" # update stats if required - self.statistics.getstats() + translatables = self.statistics.getstats()["total"] if lastitem is None: minitem = 0 else: minitem = lastitem + 1 - maxitem = len(self.transunits) + maxitem = len(translatables) validitems = range(minitem, maxitem) if search.assignedto or search.assignedaction: - assignitems = self.assigns.finditems(search) + assignitems = self.getassigns().finditems(search) validitems = [item for item in validitems if item in assignitems] # loop through, filtering on matchnames if required for item in validitems: if not search.matchnames: yield item for name in search.matchnames: - if item in self.statistics.classify[name]: + if translatables[item] in self.statistics.getstats()[name]: yield item def matchitems(self, newfile, uselocations=False): @@ -576,15 +542,19 @@ class pootlefile(Wrapper): matches.append((oldpo, None)) return matches + def getassigns(self): + if self.assigns is None: + self.assigns = pootleassigns(self) + return self.assigns + def mergeitem(self, oldpo, newpo, username, suggest=False): """merges any changes from newpo into oldpo""" unchanged = oldpo.target == newpo.target - if not suggest and (not oldpo.target or not newpo.target or oldpo.isheader() or newpo.isheader() or unchanged): oldpo.merge(newpo) - elif not unchanged: - #XXX: this is very inefficient! - for item, matchpo in enumerate(self.transunits): + else: + for item in self.statistics.getstats()["total"]: + matchpo = self.units[item] if matchpo == oldpo: strings = getattr(newpo.target, "strings", [newpo.target]) self.addsuggestion(item, strings, username) pootlefile.py @@ -480,22 +480,21 @@ class POTree: def getpofiles(self, languagecode, projectcode, poext="po"): """returns a list of po files for the project and language""" + pofilenames = [] + prefix = os.curdir + os.sep def addfiles(podir, dirname, fnames): """adds the files to the set of files for this project""" - basedirname = dirname.replace(podir, "", 1) - while basedirname.startswith(os.sep): - basedirname = basedirname.replace(os.sep, "", 1) - ponames = [] + if dirname == os.curdir: + basedirname = "" + else: + basedirname = dirname.replace(prefix, "", 1) for fname in fnames: - #check that it actually exists (to avoid problems with broken symbolic + # check that it actually exists (to avoid problems with broken symbolic # links, for example) - if not os.path.exists(os.path.join(dirname, fname)): - print "file does not exist:", os.path.join(dirname, fname) - continue + fpath = os.path.join(basedirname, fname) if fname.endswith(os.extsep+poext): - ponames.append(fname) - pofilenames.extend([os.path.join(basedirname, poname) for poname in ponames]) + pofilenames.append(fpath) def addgnufiles(podir, dirname, fnames): """adds the files to the set of files for this project""" @@ -506,12 +505,14 @@ class POTree: ponames = [fn for fn in fnames if fn.endswith(ext) and self.languagematch(languagecode, fn[:-len(ext)])] pofilenames.extend([os.path.join(basedirname, poname) for poname in ponames]) - pofilenames = [] podir = self.getpodir(languagecode, projectcode) if self.hasgnufiles(podir, languagecode) == "gnu": os.path.walk(podir, addgnufiles, podir) else: - os.path.walk(podir, addfiles, podir) + pwd = os.path.abspath(os.curdir) + os.chdir(podir) + os.path.walk(os.curdir, addfiles, None) + os.chdir(pwd) return pofilenames def getdefaultrights(self): potree.py @@ -33,6 +33,7 @@ from translate.tools import pocompile from translate.tools import pogrep from translate.search import match from translate.search import indexer +from translate.storage import statsdb, base from Pootle import statistics from Pootle import pootlefile from translate.storage import versioncontrol @@ -455,13 +456,14 @@ class TranslationProject(object): def scanpofiles(self): """sets the list of pofilenames by scanning the project directory""" self.pofilenames = self.potree.getpofiles(self.languagecode, self.projectcode, poext=self.fileext) - for pofilename in self.pofilenames: - if not pofilename in self.pofiles: - self.pofiles[pofilename] = pootlefile.pootlefile(self, pofilename) + filename_set = set(self.pofilenames) + pootlefile_set = set(self.pofiles.keys()) + # add any files that we don't have yet + for filename in filename_set.difference(pootlefile_set): + self.pofiles[filename] = pootlefile.pootlefile(self, filename) # remove any files that have been deleted since initialization - for pofilename in self.pofiles.keys(): - if not pofilename in self.pofilenames: - del self.pofiles[pofilename] + for filename in pootlefile_set.difference(filename_set): + del self.pofiles[filename] def getuploadpath(self, dirname, localfilename): """gets the path of a translation file being uploaded securely, creating directories as neccessary""" @@ -823,58 +825,60 @@ class TranslationProject(object): index = self.getindexer() pofile = self.pofiles[pofilename] # check if the pomtime in the index == the latest pomtime - pomtime = statistics.getmodtime(pofile.filename) - pofilenamequery = index.make_query([("pofilename", pofilename)], True) - pomtimequery = index.make_query([("pomtime", str(pomtime))], True) - gooditemsquery = index.make_query([pofilenamequery, pomtimequery], True) - gooditemsnum = index.get_query_result(gooditemsquery).get_matches_count() - # if there is at least one up-to-date indexing item, then the po file - # was not changed externally -> no need to update the database - if (gooditemsnum > 0) and (not items): - # nothing to be done - return - elif items: - # Update only specific items - usually single translation via the web - # interface. All other items should still be up-to-date (even with an - # older pomtime). - print "updating", self.languagecode, "index for", pofilename, "items", items - # delete the relevant items from the database - itemsquery = index.make_query([("itemno", str(itemno)) for itemno in items], False) - index.delete_doc([pofilenamequery, itemsquery]) - else: - # (items is None) - # The po file is not indexed - or it was changed externally (see - # "pofreshen" in pootlefile.py). - print "updating", self.projectcode, self.languagecode, "index for", pofilename - # delete all items of this file - index.delete_doc({"pofilename": pofilename}) - pofile.pofreshen() - if items is None: - # rebuild the whole index - items = range(len(pofile.transunits)) - addlist = [] - for itemno in items: - unit = pofile.transunits[itemno] - doc = {"pofilename": pofilename, "pomtime": str(pomtime), "itemno": str(itemno)} - if unit.hasplural(): - sources = unit.source.strings - targets = unit.target.strings - else: - sources = unit.source - targets = unit.target - orig = "\n".join(sources) - trans = "\n".join(targets) - doc["msgid"] = orig - doc["msgstr"] = trans - addlist.append(doc) - if addlist: - index.begin_transaction() - try: - for add_item in addlist: - index.index_document(add_item) - finally: - index.commit_transaction() - index.flush(optimize=optimize) + try: + pomtime = statistics.getmodtime(pofile.filename) + pofilenamequery = index.make_query([("pofilename", pofilename)], True) + pomtimequery = index.make_query([("pomtime", str(pomtime))], True) + gooditemsquery = index.make_query([pofilenamequery, pomtimequery], True) + gooditemsnum = index.get_query_result(gooditemsquery).get_matches_count() + # if there is at least one up-to-date indexing item, then the po file + # was not changed externally -> no need to update the database + if (gooditemsnum > 0) and (not items): + # nothing to be done + return + elif items: + # Update only specific items - usually single translation via the web + # interface. All other items should still be up-to-date (even with an + # older pomtime). + print "updating", self.languagecode, "index for", pofilename, "items", items + # delete the relevant items from the database + itemsquery = index.make_query([("itemno", str(itemno)) for itemno in items], False) + index.delete_doc([pofilenamequery, itemsquery]) + else: + # (items is None) + # The po file is not indexed - or it was changed externally (see + # "pofreshen" in pootlefile.py). + print "updating", self.projectcode, self.languagecode, "index for", pofilename + # delete all items of this file + index.delete_doc({"pofilename": pofilename}) + pofile.pofreshen() + if items is None: + # rebuild the whole index + items = range(pofile.statistics.getitemslen()) + addlist = [] + for itemno in items: + unit = pofile.getitem(itemno) + doc = {"pofilename": pofilename, "pomtime": str(pomtime), "itemno": str(itemno)} + if unit.hasplural(): + orig = "\n".join(unit.source.strings) + trans = "\n".join(unit.target.strings) + else: + orig = unit.source + trans = unit.target + doc["msgid"] = orig + doc["msgstr"] = trans + addlist.append(doc) + if addlist: + index.begin_transaction() + try: + for add_item in addlist: + index.index_document(add_item) + finally: + index.commit_transaction() + index.flush(optimize=optimize) + except (base.ParseError, IOError, OSError): + index.delete_doc({"pofilename": pofilename}) + print "Not indexing %s, since it is corrupt" % (pofilename,) def matchessearch(self, pofilename, search): """returns whether any items in the pofilename match the search (based on collected stats etc)""" @@ -884,9 +888,9 @@ class TranslationProject(object): # search.assignedto == [None] means assigned to nobody if search.assignedto or search.assignedaction: if search.assignedto == [None]: - assigns = self.pofiles[pofilename].assigns.getunassigned(search.assignedaction) + assigns = self.pofiles[pofilename].getassigns().getunassigned(search.assignedaction) else: - assigns = self.pofiles[pofilename].assigns.getassigns() + assigns = self.pofiles[pofilename].getassigns().getassigns() if search.assignedto is not None: if search.assignedto not in assigns: return False @@ -900,7 +904,7 @@ class TranslationProject(object): postats = self.getpostats(pofilename) matches = False for name in search.matchnames: - if postats[name]: + if postats.get(name): matches = True if not matches: return False @@ -970,7 +974,7 @@ class TranslationProject(object): continue # TODO: move this to iteritems if search.searchtext: - unit = pofile.transunits[item] + unit = pofile.getitem(item) if grepfilter.filterunit(unit): yield pofilename, item else: @@ -996,9 +1000,8 @@ class TranslationProject(object): assigncount = 0 if not usercount: return assigncount - docountwords = lambda pofilename: self.countwords([(pofilename, item) for item in range(self.pofiles[pofilename].statistics.getitemslen())]) pofilenames = [pofilename for pofilename in self.searchpofilenames(None, search, includelast=True)] - wordcounts = [(pofilename, docountwords(pofilename)) for pofilename in pofilenames] + wordcounts = [(pofilename, self.getpofile(pofilename).statistics.getquickstats()['totalsourcewords']) for pofilename in pofilenames] totalwordcount = sum([wordcount for pofilename, wordcount in wordcounts]) wordsperuser = totalwordcount / usercount @@ -1007,21 +1010,23 @@ class TranslationProject(object): userwords = 0 for pofilename, wordcount in wordcounts: pofile = self.getpofile(pofilename) + sourcewordcount = pofile.statistics.getunitstats()['sourcewordcount'] for item in pofile.iteritems(search, None): # TODO: move this to iteritems if search.searchtext: validitem = False - unit = pofile.transunits[item] + unit = pofile.getitem(item) if grepfilter.filterunit(unit): validitem = True if not validitem: continue - itemwordcount = self.countwords([(pofilename, item)]) + itemwordcount = sourcewordcount[item] + #itemwordcount = statsdb.wordcount(str(pofile.getitem(item).source)) if userwords + itemwordcount > wordsperuser: usernum = min(usernum+1, len(assignto)-1) userwords = 0 userwords += itemwordcount - pofile.assigns.assignto(item, assignto[usernum], action) + pofile.getassigns().assignto(item, assignto[usernum], action) assigncount += 1 return assigncount @@ -1037,12 +1042,12 @@ class TranslationProject(object): for item in pofile.iteritems(search, None): # TODO: move this to iteritems if search.searchtext: - unit = pofile.transunits[item] + unit = pofile.getitem(item) if grepfilter.filterunit(unit): - pofile.assigns.unassign(item, assignedto, action) + pofile.getassigns().unassign(item, assignedto, action) assigncount += 1 else: - pofile.assigns.unassign(item, assignedto, action) + pofile.getassigns().unassign(item, assignedto, action) assigncount += 1 return assigncount @@ -1100,7 +1105,6 @@ class TranslationProject(object): alltotal += total for pofilename in slowfiles: self.pofiles[pofilename].statistics.updatequickstats(save=False) - self.savequickstats() translatedwords, translated, fuzzywords, fuzzy, totalwords, total = self.quickstats[pofilename] alltranslatedwords += translatedwords alltranslated += translated @@ -1108,43 +1112,56 @@ class TranslationProject(object): allfuzzy += fuzzy alltotalwords += totalwords alltotal += total + if slowfiles: + self.savequickstats() return {"translatedsourcewords": alltranslatedwords, "translated": alltranslated, "fuzzysourcewords": allfuzzywords, "fuzzy": allfuzzy, "totalsourcewords": alltotalwords, "total": alltotal} def combinestats(self, pofilenames=None): """combines translation statistics for the given po files (or all if None given)""" - totalstats = {} if pofilenames is None: pofilenames = self.pofilenames + pofilenames = [pofilename for pofilename in pofilenames + if pofilename != None and not os.path.isdir(pofilename)] + total_stats = self.combine_totals(pofilenames) + total_stats['units'] = self.combine_unit_stats(pofilenames) + total_stats['assign'] = self.combineassignstats(pofilenames) + return total_stats + + def combine_totals(self, pofilenames): + totalstats = {} + for pofilename in pofilenames: + pototals = self.getpototals(pofilename) + for name, items in pototals.iteritems(): + totalstats[name] = totalstats.get(name, 0) + pototals[name] + return totalstats + + def combine_unit_stats(self, pofilenames): + unit_stats = {} for pofilename in pofilenames: - if not pofilename or os.path.isdir(pofilename): - continue postats = self.getpostats(pofilename) for name, items in postats.iteritems(): - totalstats[name] = totalstats.get(name, []) + [(pofilename, item) for item in items] - assignstats = self.combineassignstats(pofilenames) - totalstats.update(assignstats) - return totalstats + unit_stats.setdefault(name, []).extend([(pofilename, item) for item in items]) + return unit_stats def combineassignstats(self, pofilenames=None, action=None): """combines assign statistics for the given po files (or all if None given)""" - totalstats = {} - if pofilenames is None: - pofilenames = self.pofilenames + assign_stats = {} for pofilename in pofilenames: assignstats = self.getassignstats(pofilename, action) for name, items in assignstats.iteritems(): - totalstats["assign-"+name] = totalstats.get("assign-"+name, []) + [(pofilename, item) for item in items] - return totalstats + assign_stats.setdefault(name, []).extend([(pofilename, item) for item in items]) + return assign_stats def countwords(self, stats): """counts the number of words in the items represented by the stats list""" wordcount = 0 for pofilename, item in stats: pofile = self.pofiles[pofilename] - if 0 <= item < len(pofile.statistics.sourcewordcounts): - wordcount += sum(pofile.statistics.sourcewordcounts[item]) + if 0 <= item < len(pofile.statistics.getunitstats()['sourcewordcount']): + wordcount += pofile.statistics.getunitstats()['sourcewordcount'][item] + print "projects::countwords()" return wordcount def getpomtime(self): @@ -1175,10 +1192,16 @@ class TranslationProject(object): """calculates translation statistics for the given po file""" return self.pofiles[pofilename].statistics.getstats() + def getpototals(self, pofilename): + """calculates translation statistics for the given po file""" + return self.pofiles[pofilename].statistics.getquickstats() + def getassignstats(self, pofilename, action=None): """calculates translation statistics for the given po file (can filter by action if given)""" - polen = len(self.getpostats(pofilename)["total"]) - assigns = self.pofiles[pofilename].assigns.getassigns() + polen = self.getpototals(pofilename).get("total", 0) + # Temporary code to avoid traceback. Was: +# polen = len(self.getpostats(pofilename)["total"]) + assigns = self.pofiles[pofilename].getassigns().getassigns() assignstats = {} for username, userassigns in assigns.iteritems(): allitems = [] @@ -1198,14 +1221,13 @@ class TranslationProject(object): def getpofilelen(self, pofilename): """returns number of items in the given pofilename""" - # TODO: needn't parse the file for this ... pofile = self.getpofile(pofilename) - return len(pofile.transunits) + return pofile.statistics.getitemslen() def getitems(self, pofilename, itemstart, itemstop): """returns a set of items from the pofile, converted to original and translation strings""" pofile = self.getpofile(pofilename) - units = pofile.transunits[max(itemstart,0):itemstop] + units = [pofile.units[index] for index in pofile.statistics.getstats()["total"][max(itemstart,0):itemstop]] return units def updatetranslation(self, pofilename, item, newvalues, session): @@ -1318,7 +1340,7 @@ class TranslationProject(object): if isinstance(pofile, (str, unicode)): pofilename = pofile pofile = self.getpofile(pofilename) - return termmatcher.matches(pofile.transunits[item].source) + return termmatcher.matches(pofile.getitem(item).source) except Exception, e: session.server.errorhandler.logerror(traceback.format_exc()) return [] projects.py @@ -3,235 +3,75 @@ from translate.storage import statsdb from translate.filters import checks from translate.misc.multistring import multistring -def getmodtime(filename, default=None): - """gets the modificationtime of the given file""" - if os.path.exists(filename): - return os.stat(filename)[os.path.stat.ST_MTIME] - else: - return default +STATS_DB_FILE = None -class StatsFile: - """Manages a statistics data file""" - def __init__(self, basefile): - self.basefile = basefile - self.refname = self.basefile.filename - self.filename = self.refname + os.extsep + "stats" - - def read(self): - """return the contents of the stats file""" - sfile = open(self.filename, "r") - contents = sfile.read() - sfile.close() - return contents - - def save(self, statsstring): - """save the stats data""" - sfile = open(self.filename, "w") - if os.path.exists(self.basefile.pendingfilename): - sfile.write("%d %d\n" % (getmodtime(self.basefile.filename), getmodtime(self.basefile.pendingfilename))) - else: - sfile.write("%d\n" % getmodtime(self.basefile.filename)) - sfile.write(statsstring) - sfile.close() - - def hasparent(self): - """check if the stats file has a parent data file, if not delete it""" - if os.path.exists(self.basefile.filename): - return True - else: - if os.path.exists(self.filename): - os.remove(self.filename) - return False +def getmodtime(filename): + return statsdb.get_mod_info(filename, errors_return_empty=True, empty_return=None) class pootlestatistics: """this represents the statistics known about a file""" - def __init__(self, basefile, generatestats=True): + def __init__(self, basefile): """constructs statistic object for the given file""" # TODO: try and remove circular references between basefile and this class self.basefile = basefile - self.sfile = StatsFile(self.basefile) - self.classify = {} - self.sourcewordcounts = [] - self.targetwordcounts = [] - if generatestats: - self.getstats() - - def getstats(self): + self._stats = None + self._totals = None + self._unitstats = None + self.statscache = statsdb.StatsCache(STATS_DB_FILE) + + def getquickstats(self): + """returns the quick statistics (totals only)""" + if not self._totals: + self._totals = self.statscache.filetotals(self.basefile.filename, errors_return_empty=True) + return self._totals + + def getstats(self, checker=None): """reads the stats if neccessary or returns them from the cache""" - if os.path.exists(self.sfile.filename): - try: - self.readstats() - except Exception, e: - print "Error reading stats from %s, so recreating (Error was %s)" % (self.sfile.filename, e) - self.statspomtime = None - raise - pomtime = getmodtime(self.basefile.filename) - pendingmtime = getmodtime(self.basefile.pendingfilename, None) - if hasattr(self, "pendingmtime"): - self.basefile.readpendingfile() - lastpomtime = getattr(self, "statspomtime", None) - lastpendingmtime = getattr(self, "statspendingmtime", None) - if pomtime is None or pomtime != lastpomtime or pendingmtime != lastpendingmtime: - self.calcstats() - self.savestats() - return self.stats - - def readstats(self): - """reads the stats from the associated stats file, setting the required variables""" - statsmtime = getmodtime(self.sfile.filename) - if statsmtime == getattr(self, "statsmtime", None): - return - stats = self.sfile.read() - mtimes, statsstring = stats.split("\n", 1) - mtimes = mtimes.strip().split() - if len(mtimes) == 1: - frompomtime = int(mtimes[0]) - frompendingmtime = None - elif len(mtimes) == 2: - frompomtime = int(mtimes[0]) - frompendingmtime = int(mtimes[1]) - stats = {} - for line in statsstring.split("\n"): - if not line.strip(): - continue - if not ":" in line: - print "invalid stats line in", self.sfile.filename,line - continue - name, items = line.split(":", 1) - if name == "msgidwordcounts" or name == "sourcewordcounts": - stats["sourcewordcounts"] = [[int(subitem.strip()) for subitem in item.strip().split("/")] for item in items.strip().split(",") if item] - elif name == "msgstrwordcounts" or name == "targetwordcounts": - stats["targetwordcounts"] = [[int(subitem.strip()) for subitem in item.strip().split("/")] for item in items.strip().split(",") if item] - else: - items = [int(item.strip()) for item in items.strip().split(",") if item] - stats[name.strip()] = items - # save all the read times, data simultaneously - self.statspomtime, self.statspendingmtime, self.statsmtime, self.stats, self.sourcewordcounts, self.targetwordcounts = frompomtime, frompendingmtime, statsmtime, stats, stats["sourcewordcounts"], stats["targetwordcounts"] - # if in old-style format (counts instead of items), recalculate - totalitems = stats.get("total", []) - if len(totalitems) == 1 and totalitems[0] != 0: - self.calcstats() - self.savestats() - if (len(stats["sourcewordcounts"]) < len(totalitems)) or (len(stats["targetwordcounts"]) < len(totalitems)): - self.basefile.pofreshen() - self.countwords() - self.savestats() - - def savestats(self): - """saves the current statistics to file""" - if not self.sfile.hasparent(): - return - # assumes self.stats is up to date - try: - statsstring = "\n".join(["%s:%s" % (name, ",".join(map(str,items))) for name, items in self.stats.iteritems()]) - wordcountsstring = "sourcewordcounts:" + ",".join(["/".join(map(str,subitems)) for subitems in self.sourcewordcounts]) - wordcountsstring += "\ntargetwordcounts:" + ",".join(["/".join(map(str,subitems)) for subitems in self.targetwordcounts]) - self.sfile.save(statsstring + "\n" + wordcountsstring) - except IOError: - # TODO: log a warning somewhere. we don't want an error as this is an optimization - pass - self.updatequickstats() + if checker == None: + checker = self.basefile.checker + if not self._stats: + self._stats = self.statscache.filestats(self.basefile.filename, checker, errors_return_empty=True) + return self._stats + + def purge_totals(self): + """Temporary helper to clean up where needed. We might be able to remove + this after the move to cpo.""" + self._totals = None + self._stats = None + self._unitstats = None + + def getunitstats(self): + if not self._unitstats: + self._unitstats = self.statscache.unitstats(self.basefile.filename, errors_return_empty=True) + return self._unitstats def updatequickstats(self, save=True): """updates the project's quick stats on this file""" - translated = self.stats.get("translated") - fuzzy = self.stats.get("fuzzy") - translatedwords = sum([sum(self.sourcewordcounts[item]) for item in translated if 0 <= item < len(self.sourcewordcounts)]) - fuzzywords = sum([sum(self.sourcewordcounts[item]) for item in fuzzy if 0 <= item < len(self.sourcewordcounts)]) - totalwords = sum([sum(partcounts) for partcounts in self.sourcewordcounts]) + totals = self.getquickstats() self.basefile.project.updatequickstats(self.basefile.pofilename, - translatedwords, len(translated), - fuzzywords, len(fuzzy), - totalwords, len(self.sourcewordcounts), + totals.get("translatedsourcewords", 0), totals.get("translated", 0), + totals.get("fuzzysourcewords", 0), totals.get("fuzzy", 0), + totals.get("totalsourcewords", 0), totals.get("total", 0), save) - def calcstats(self): - """calculates translation statistics for the given file""" - # handle this being called when self.basefile.statistics is being set and calcstats is called from self.__init__ - if not hasattr(self.basefile, "statistics"): - self.basefile.statistics = self - self.basefile.pofreshen() - self.stats = dict([(name, items) for name, items in self.classify.iteritems()]) - - def classifyunit(self, unit): - """returns all classify keys that the unit should match""" - classes = ["total"] - if unit.isfuzzy(): - classes.append("fuzzy") - if unit.gettargetlen() == 0: - classes.append("blank") - if unit.istranslated(): - classes.append("translated") - checkresult = self.basefile.checker.run_filters(unit) - for checkname, checkmessage in checkresult.iteritems(): - classes.append("check-" + checkname) - return classes - - def classifyunits(self): - """makes a dictionary of which units fall into which classifications""" - self.classify = {} - self.classify["fuzzy"] = [] - self.classify["blank"] = [] - self.classify["translated"] = [] - self.classify["has-suggestion"] = [] - self.classify["total"] = [] - for checkname in self.basefile.checker.getfilters().keys(): - self.classify["check-" + checkname] = [] - for item, unit in enumerate(self.basefile.transunits): - classes = self.classifyunit(unit) - if self.basefile.getsuggestions(item): - classes.append("has-suggestion") - for classname in classes: - if classname in self.classify: - self.classify[classname].append(item) - else: - self.classify[classname] = item - self.countwords() - - def countwords(self): - """counts the words in each of the units""" - self.sourcewordcounts = [] - self.targetwordcounts = [] - for unit in self.basefile.transunits: - if isinstance(unit.source, multistring): - sourcestrings = unit.source.strings - else: - sourcestrings = [unit.source] - if isinstance(unit.target, multistring): - targetstrings = unit.target.strings - else: - targetstrings = [unit.target] - - self.sourcewordcounts.append([statsdb.wordcount(text) for text in sourcestrings]) - self.targetwordcounts.append([statsdb.wordcount(text) for text in targetstrings]) - def reclassifyunit(self, item): - """updates the classification of a unit in self.classify""" - unit = self.basefile.transunits[item] - self.sourcewordcounts[item] = [statsdb.wordcount(text) for text in unit.source.strings] - self.targetwordcounts[item] = [statsdb.wordcount(text) for text in unit.target.strings] - classes = self.classifyunit(unit) - if self.basefile.getsuggestions(item): - classes.append("has-suggestion") - for classname, matchingitems in self.classify.items(): + """Reclassifies all the information in the database and self._stats about + the given unit""" + unit = self.basefile.getitem(item) + item = self.getstats()["total"][item] + + classes = self.statscache.recacheunit(self.basefile.filename, self.basefile.checker, unit) + for classname, matchingitems in self.getstats().items(): if (classname in classes) != (item in matchingitems): if classname in classes: - self.classify[classname].append(item) + self.getstats()[classname].append(item) else: - self.classify[classname].remove(item) - self.classify[classname].sort() - self.calcstats() - self.savestats() + self.getstats()[classname].remove(item) + self.getstats()[classname].sort() + # We should be doing better, but for now it is easiet to simply force a + # reload from the database + self.purge_totals() def getitemslen(self): """gets the number of items in the file""" - # TODO: simplify this, and use wherever its needed - if hasattr(self.basefile, "transunits"): - return len(self.basefile.transunits) - elif hasattr(self, "stats") and "total" in self.stats: - return len(self.stats["total"]) - elif hasattr(self, "classify") and "total" in self.classify: - return len(self.classify["total"]) - else: - # we hadn't read stats... - return len(self.getstats()["total"]) + return self.getquickstats()["total"] statistics.py @@ -44,7 +44,7 @@ </div> </div> <div id="itemstats" class="item-statistics"> - <span py:replace="XML(item.stats.summary)"> + <span py:if="item.stats.summary" py:replace="XML(item.stats.summary)"> 2/2 words (100%) translated <span class="string-statistics">[2/2 strings]</span> </span> <span py:for="check in item.stats.checks" py:strip="True"> templates/navbar.html @@ -87,23 +87,6 @@ msgstr ""''' assert unit.source == "test" assert unit.target == "rest" - def test_classify(self): - """Test basic classification""" - posource = 'msgid "test"\nmsgstr ""\n' - pofile = self.poparse(posource) - pofile.project.checker = checks.TeeChecker() - unit = pofile.units[0] - classify = pofile.statistics.classifyunit - classes = classify(unit) - assert 'blank' in classes - unit.target = "Gom" - classes = classify(unit) - assert 'translated' in classes - assert 'blank' not in classes - unit.markfuzzy() - classes = classify(unit) - assert 'fuzzy' in classes - def test_classifyunits(self): "Tests basic use of classifyunits." posource = r'''#: test.c @@ -117,15 +100,11 @@ msgstr "tafel" msgid "chair" msgstr ""''' pofile = self.poparse(posource) - pofile.transunits = [poel for poel in pofile.units if not (poel.isheader() or poel.isblank())] - pofile.statistics.classifyunits() - classify = pofile.statistics.classify - print classify - for i in pofile.units: - print str(i) - assert classify['fuzzy'] == [1] - assert classify['blank'] == [2] - assert len(classify['total']) == 3 + pofile.savepofile() + assert pofile.statistics.getstats()['fuzzy'] == [1] + assert pofile.statistics.getstats()['untranslated'] == [2] + assert pofile.statistics.getstats()['translated'] == [0] + assert pofile.statistics.getstats()['total'] == [0, 1, 2] def test_updateunit(self): """Test the updateunit() method.""" test_pootlefile.py @@ -98,9 +98,9 @@ class TranslatePage(pagelayout.PootleNavPage): icon="edit" if self.pofilename is not None: postats = self.project.getpostats(self.pofilename) - blank, fuzzy = postats["blank"], postats["fuzzy"] + untranslated, fuzzy = postats["untranslated"], postats["fuzzy"] translated, total = postats["translated"], postats["total"] - mainstats = self.localize("%d/%d translated\n(%d blank, %d fuzzy)", len(translated), len(total), len(blank), len(fuzzy)) + mainstats = self.localize("%d/%d translated\n(%d untranslated, %d fuzzy)", len(translated), len(total), len(untranslated), len(fuzzy)) pagelinks = self.getpagelinks("?translate=1&view=1", rows) navbarpath_dict = self.makenavbarpath_dict(self.project, self.session, self.pofilename, dirfilter=self.dirfilter or "") # templatising @@ -354,7 +354,7 @@ class TranslatePage(pagelayout.PootleNavPage): """returns any checker filters the user has asked to match...""" matchnames = [] for checkname in self.argdict: - if checkname in ["fuzzy", "blank", "translated", "has-suggestion"]: + if checkname in ["fuzzy", "untranslated", "translated"]: matchnames.append(checkname) elif checkname in checker.getfilters(): matchnames.append("check-" + checkname) translatepage.py test_statistics.py 8f96ef12462bbb5bcf3b490bb7db5be4fd9b2ece winterstream winterstream@54714841-351b-0410-a198-e36a94b762f5 http://github.com/julen/pootle/commit/49ca02f88f35f99eade40c044f4d6c98f264b63d 49ca02f88f35f99eade40c044f4d6c98f264b63d 2008-07-01T04:23:36-07:00 2008-07-01T04:23:36-07:00 Merged in changes from branches/Pootle-diet-and-trunk-merge. Please see the branch if you are interested in the changes that led to this commit. git-svn-id: https://translate.svn.sourceforge.net/svnroot/translate/src/trunk/Pootle@7686 54714841-351b-0410-a198-e36a94b762f5 33d5ed133e1d790ed8b735f1a21e137c8711acf7 winterstream winterstream@54714841-351b-0410-a198-e36a94b762f5