#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# Copyright 2004-2006 Zuza Software Foundation
#
# This file is part of translate.
#
# translate is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# translate is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with translate; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
"""manages projects and files and translations"""
from translate.storage import factory
from translate.filters import checks
from translate.convert import po2csv
from translate.convert import po2xliff
from translate.convert import xliff2po
from translate.convert import po2ts
from translate.convert import pot2po
from translate.convert import po2oo
from translate.tools import pocompile
from translate.tools import pogrep
from translate.search import match
from translate.search import indexer
from translate.storage import statsdb, base
from Pootle import statistics
from Pootle import pootlefile
from translate.storage import versioncontrol
from jToolkit import timecache
from jToolkit import prefs
import time
import os
import cStringIO
import traceback
import gettext
class RightsError(ValueError):
pass
class InternalAdminSession:
"""A fake session used for doing internal admin jobs"""
def __init__(self):
self.username = "internal"
self.isopen = True
def localize(self, message):
return message
def issiteadmin(self):
return True
class potimecache(timecache.timecache):
"""Caches pootlefile objects, remembers time, and reverts back to statistics when necessary..."""
def __init__(self, expiryperiod, project):
"""initialises the cache to keep objects for the given expiryperiod, and point back to the project"""
timecache.timecache.__init__(self, expiryperiod)
self.project = project
def __getitem__(self, key):
"""[] access of items"""
if key and not dict.__contains__(self, key):
popath = os.path.join(self.project.podir, key)
if os.path.exists(popath):
# update the index to pofiles...
self.project.scanpofiles()
return timecache.timecache.__getitem__(self, key)
def expire(self, pofilename):
"""expires the given pofilename by recreating it (holding only stats)"""
timestamp, currentfile = dict.__getitem__(self, pofilename)
if currentfile.pomtime is not None:
# use the currentfile.pomtime as a timestamp as well, so any modifications will extend its life
if time.time() - currentfile.pomtime > self.expiryperiod.seconds:
self.__setitem__(pofilename, pootlefile.pootlefile(self.project, pofilename))
class TranslationProject(object):
"""Manages iterating through the translations in a particular project"""
fileext = "po"
index_directory = ".translation_index"
def __init__(self, languagecode, projectcode, potree, create=False):
self.languagecode = languagecode
self.projectcode = projectcode
self.potree = potree
self.languagename = self.potree.getlanguagename(self.languagecode)
self.projectname = self.potree.getprojectname(self.projectcode)
self.projectdescription = self.potree.getprojectdescription(self.projectcode)
self.pofiles = potimecache(15*60, self)
self.projectcheckerstyle = self.potree.getprojectcheckerstyle(self.projectcode)
checkerclasses = [checks.projectcheckers.get(self.projectcheckerstyle, checks.StandardChecker), checks.StandardUnitChecker]
self.checker = checks.TeeChecker(checkerclasses=checkerclasses, errorhandler=self.filtererrorhandler, languagecode=languagecode)
self.fileext = self.potree.getprojectlocalfiletype(self.projectcode)
self.quickstats = {}
# terminology matcher
self.termmatcher = None
self.termmatchermtime = None
if create:
self.converttemplates(InternalAdminSession())
self.podir = potree.getpodir(languagecode, projectcode)
if self.potree.hasgnufiles(self.podir, self.languagecode) == "gnu":
self.filestyle = "gnu"
else:
self.filestyle = "std"
self.readprefs()
self.scanpofiles()
self.readquickstats()
self.initindex()
def readprefs(self):
"""reads the project preferences"""
self.prefs = prefs.PrefsParser()
self.prefsfile = os.path.join(self.podir, "pootle-%s-%s.prefs" % (self.projectcode, self.languagecode))
if not os.path.exists(self.prefsfile):
prefsfile = open(self.prefsfile, "w")
prefsfile.write("# Pootle preferences for project %s, language %s\n\n" % (self.projectcode, self.languagecode))
prefsfile.close()
self.prefs.parsefile(self.prefsfile)
def saveprefs(self):
"""saves the project preferences"""
self.prefs.savefile()
def getrightnames(self, session):
"""gets the available rights and their localized names"""
localize = session.localize
# l10n: Verb
return [("view", localize("View")),
("suggest", localize("Suggest")),
("translate", localize("Translate")),
("overwrite", localize("Overwrite")),
# l10n: Verb
("review", localize("Review")),
# l10n: Verb
("archive", localize("Archive")),
# l10n: This refers to generating the binary .mo file
("pocompile", localize("Compile PO files")),
("assign", localize("Assign")),
("admin", localize("Administrate")),
("commit", localize("Commit")),
]
def getrights(self, session=None, username=None, usedefaults=True):
"""gets the rights for the given user (name or session, or not-logged-in if username is None)
if usedefaults is False then None will be returned if no rights are defined (useful for editing rights)"""
# internal admin sessions have all rights
if isinstance(session, InternalAdminSession):
return [right for right, localizedright in self.getrightnames(session)]
if session is not None and session.isopen and username is None:
username = session.username
if username is None:
username = "nobody"
rights = None
rightstree = getattr(self.prefs, "rights", None)
if rightstree is not None:
if rightstree.__hasattr__(username):
rights = rightstree.__getattr__(username)
else:
rights = None
if rights is None:
if usedefaults:
if username == "nobody":
rights = "view"
elif rightstree is None:
if self.languagecode == "en":
rights = "view, archive, pocompile"
else:
rights = self.potree.getdefaultrights()
else:
rights = getattr(rightstree, "default", None)
else:
return rights
rights = [right.strip() for right in rights.split(",")]
if session is not None and session.issiteadmin():
if "admin" not in rights:
rights.append("admin")
return rights
def getuserswithinterest(self, session):
"""returns all the users who registered for this language and project"""
def usableuser(user, userprefs):
if user in ["__dummy__", "default", "nobody"]:
return False
return self.languagecode in getattr(userprefs, "languages", [])
users = {}
for username, userprefs in session.loginchecker.users.iteritems():
if usableuser(username, userprefs):
# Let's build a nice descriptive name for use in the interface. It will
# contain both the username and the full name, if available.
name = getattr(userprefs, "name", None)
if name:
description = "%s (%s)" % (name, username)
else:
description = username
setattr(userprefs, "description", description)
users[username] = userprefs
return users
def getuserswithrights(self):
"""gets all users that have rights defined for this project"""
return [username for username, user_rights in getattr(self.prefs, "rights", {}).iteritems()]
def setrights(self, username, rights):
"""sets the rights for the given username... (or not-logged-in if username is None)"""
if username is None: username = "nobody"
if isinstance(rights, list):
rights = ", ".join(rights)
if not hasattr(self.prefs, "rights"):
self.prefs.rights = prefs.PrefNode(self.prefs, "rights")
self.prefs.rights.__setattr__(username, rights)
self.saveprefs()
def delrights(self, session, username):
"""deletes teh rights for the given username"""
# l10n: Don't translate "nobody" or "default"
if username == "nobody" or username == "default":
# l10n: Don't translate "nobody" or "default"
raise RightsError(session.localize('You cannot remove the "nobody" or "default" user'))
self.prefs.rights.__delattr__(username)
self.saveprefs()
def getgoalnames(self):
"""gets the goals and associated files for the project"""
goals = getattr(self.prefs, "goals", {})
goallist = []
for goalname, goalnode in goals.iteritems():
goallist.append(goalname.decode("utf-8"))
goallist.sort()
return goallist
def getgoals(self):
"""gets the goal, goalnode tuples"""
goals = getattr(self.prefs, "goals", {})
newgoals = {}
for goalname, goalnode in goals.iteritems():
newgoals[goalname.decode("utf-8")] = goalnode
return newgoals
def getgoalfiles(self, goalname, dirfilter=None, maxdepth=None, includedirs=True, expanddirs=False, includepartial=False):
"""gets the files for the given goal, with many options!
dirfilter limits to files in a certain subdirectory
maxdepth limits to files / directories of a certain depth
includedirs specifies whether to return directory names
expanddirs specifies whether to expand directories and return all files in them
includepartial specifies whether to return directories that are not in the goal, but have files below maxdepth in the goal"""
if not goalname:
return self.getnogoalfiles(dirfilter=dirfilter, maxdepth=maxdepth, includedirs=includedirs, expanddirs=expanddirs)
goals = getattr(self.prefs, "goals", {})
poext = os.extsep + self.fileext
pathsep = os.path.sep
unique = lambda filelist: dict.fromkeys(filelist).keys()
for testgoalname, goalnode in self.getgoals().iteritems():
if goalname != testgoalname: continue
goalmembers = getattr(goalnode, "files", "")
goalmembers = [goalfile.strip() for goalfile in goalmembers.split(",") if goalfile.strip()]
goaldirs = [goaldir for goaldir in goalmembers if goaldir.endswith(pathsep)]
goalfiles = [goalfile for goalfile in goalmembers if not goalfile.endswith(pathsep) and goalfile in self.pofilenames]
if expanddirs:
expandgoaldirs = []
expandgoalfiles = []
for goaldir in goaldirs:
expandedfiles = self.browsefiles(dirfilter=goaldir, includedirs=includedirs, includefiles=True)
expandgoalfiles.extend([expandfile for expandfile in expandedfiles if expandfile.endswith(poext)])
expandgoaldirs.extend([expanddir + pathsep for expanddir in expandedfiles if not expanddir.endswith(poext)])
goaldirs = unique(goaldirs + expandgoaldirs)
goalfiles = unique(goalfiles + expandgoalfiles)
if dirfilter:
if not dirfilter.endswith(pathsep) and not dirfilter.endswith(poext):
dirfilter += pathsep
goalfiles = [goalfile for goalfile in goalfiles if goalfile.startswith(dirfilter)]
goaldirs = [goaldir for goaldir in goaldirs if goaldir.startswith(dirfilter)]
if maxdepth is not None:
if includepartial:
partialdirs = [goalfile for goalfile in goalfiles if goalfile.count(pathsep) > maxdepth]
partialdirs += [goalfile for goalfile in goaldirs if goalfile.count(pathsep) > maxdepth]
makepartial = lambda goalfile: pathsep.join(goalfile.split(pathsep)[:maxdepth+1])+pathsep
partialdirs = [makepartial(goalfile) for goalfile in partialdirs]
goalfiles = [goalfile for goalfile in goalfiles if goalfile.count(pathsep) <= maxdepth]
goaldirs = [goaldir for goaldir in goaldirs if goaldir.count(pathsep) <= maxdepth+1]
if includepartial:
goaldirs += partialdirs
if includedirs:
return unique(goalfiles + goaldirs)
else:
return unique(goalfiles)
return []
def getnogoalfiles(self, dirfilter=None, maxdepth=None, includedirs=True, expanddirs=False):
"""Returns the files that are not in a goal. This works with getgoalfiles
and therefre the API resembles that closely"""
all = self.browsefiles(dirfilter=dirfilter, maxdepth=maxdepth, includedirs=includedirs)
pathsep = os.path.sep
for testgoalname in self.getgoals():
goalfiles = self.getgoalfiles(testgoalname, dirfilter=dirfilter, maxdepth=maxdepth, includedirs=includedirs, expanddirs=expanddirs, includepartial=False)
for goalfile in goalfiles:
if goalfile.endswith(pathsep):
goalfile = goalfile[:-len(pathsep)]
all.remove(goalfile)
return all
def getancestry(self, filename):
"""returns parent directories of the file"""
ancestry = []
parts = filename.split(os.path.sep)
for i in range(1, len(parts)):
ancestor = os.path.join(*parts[:i]) + os.path.sep
ancestry.append(ancestor)
return ancestry
def getfilegoals(self, filename):
"""gets the goals the given file is part of"""
goals = self.getgoals()
filegoals = []
ancestry = self.getancestry(filename)
for goalname, goalnode in goals.iteritems():
goalfiles = getattr(goalnode, "files", "")
goalfiles = [goalfile.strip() for goalfile in goalfiles.split(",") if goalfile.strip()]
if filename in goalfiles:
filegoals.append(goalname)
continue
for ancestor in ancestry:
if ancestor in goalfiles:
filegoals.append(goalname)
continue
return filegoals
def setfilegoals(self, session, goalnames, filename):
"""sets the given file to belong to the given goals exactly"""
filegoals = self.getfilegoals(filename)
for othergoalname in filegoals:
if othergoalname not in goalnames:
self.removefilefromgoal(session, othergoalname, filename)
for goalname in goalnames:
goalfiles = self.getgoalfiles(goalname)
if filename not in goalfiles:
goalfiles.append(filename)
self.setgoalfiles(session, goalname, goalfiles)
def removefilefromgoal(self, session, goalname, filename):
"""removes the given file from the goal"""
goalfiles = self.getgoalfiles(goalname)
if filename in goalfiles:
goalfiles.remove(filename)
self.setgoalfiles(session, goalname, goalfiles)
else:
unique = lambda filelist: dict.fromkeys(filelist).keys()
ancestry = self.getancestry(filename)
for ancestor in ancestry:
if ancestor in goalfiles:
filedepth = filename.count(os.path.sep)
ancestordirs = self.getgoalfiles(goalname, ancestor, maxdepth=filedepth+1, includedirs=True, expanddirs=True)
ancestordirs = [ancestorfile for ancestorfile in ancestordirs if ancestorfile.endswith(os.path.sep)]
if filename.endswith(os.path.sep):
ancestorfiles = self.getgoalfiles(goalname, ancestor, maxdepth=filedepth-1, expanddirs=True)
else:
ancestorfiles = self.getgoalfiles(goalname, ancestor, maxdepth=filedepth, expanddirs=True)
ancestorfiles = unique(ancestordirs + ancestorfiles)
if not filename in ancestorfiles:
raise KeyError("expected to find file %s in ancestor %s files %r" % (filename, ancestor, ancestorfiles))
ancestorfiles.remove(filename)
ancestorfiles.remove(ancestor)
goalfiles.remove(ancestor)
goalfiles.extend(ancestorfiles)
self.setgoalfiles(session, goalname, goalfiles)
continue
def setgoalfiles(self, session, goalname, goalfiles):
"""sets the goalfiles for the given goalname"""
if "admin" not in self.getrights(session):
raise RightsError(session.localize("You do not have rights to alter goals here"))
if isinstance(goalfiles, list):
goalfiles = [goalfile.strip() for goalfile in goalfiles if goalfile.strip()]
goalfiles.sort()
goalfiles = ", ".join(goalfiles)
if not hasattr(self.prefs, "goals"):
self.prefs.goals = prefs.PrefNode(self.prefs, "goals")
goals = self.getgoals()
goalname = goalname.encode("utf-8")
if not goalname in goals:
# TODO: check that its a valid goalname (alphanumeric etc)
self.prefs.goals.__setattr__(goalname, prefs.PrefNode(self.prefs.goals, goalname))
goalnode = self.prefs.goals.__getattr__(goalname)
goalnode.files = goalfiles
self.saveprefs()
def getgoalusers(self, goalname):
"""gets the users for the given goal"""
goals = self.getgoals()
for testgoalname, goalnode in goals.iteritems():
if goalname != testgoalname: continue
goalusers = getattr(goalnode, "users", "")
goalusers = [goaluser.strip() for goaluser in goalusers.split(",") if goaluser.strip()]
return goalusers
return []
def getusergoals(self, username):
"""gets the goals the given user is part of"""
goals = getattr(self.prefs, "goals", {})
usergoals = []
for goalname, goalnode in goals.iteritems():
goalusers = getattr(goalnode, "users", "")
goalusers = [goaluser.strip() for goaluser in goalusers.split(",") if goaluser.strip()]
if username in goalusers:
usergoals.append(goalname)
continue
return usergoals
def addusertogoal(self, session, goalname, username, exclusive=False):
"""adds the given user to the goal"""
if exclusive:
usergoals = self.getusergoals(username)
for othergoalname in usergoals:
if othergoalname != goalname:
self.removeuserfromgoal(session, othergoalname, username)
goalusers = self.getgoalusers(goalname)
if username not in goalusers:
goalusers.append(username)
self.setgoalusers(session, goalname, goalusers)
def removeuserfromgoal(self, session, goalname, username):
"""removes the given user from the goal"""
goalusers = self.getgoalusers(goalname)
if username in goalusers:
goalusers.remove(username)
self.setgoalusers(session, goalname, goalusers)
def setgoalusers(self, session, goalname, goalusers):
"""sets the goalusers for the given goalname"""
if isinstance(goalname, unicode):
goalname = goalname.encode('utf-8')
if "admin" not in self.getrights(session):
raise RightsError(session.localize("You do not have rights to alter goals here"))
if isinstance(goalusers, list):
goalusers = [goaluser.strip() for goaluser in goalusers if goaluser.strip()]
goalusers = ", ".join(goalusers)
if not hasattr(self.prefs, "goals"):
self.prefs.goals = prefs.PrefNode(self.prefs, "goals")
if not hasattr(self.prefs.goals, goalname):
self.prefs.goals.__setattr__(goalname, prefs.PrefNode(self.prefs.goals, goalname))
goalnode = self.prefs.goals.__getattr__(goalname)
goalnode.users = goalusers
self.saveprefs()
def scanpofiles(self):
"""sets the list of pofilenames by scanning the project directory"""
self.pofilenames = self.potree.getpofiles(self.languagecode, self.projectcode, poext=self.fileext)
filename_set = set(self.pofilenames)
pootlefile_set = set(self.pofiles.keys())
# add any files that we don't have yet
for filename in filename_set.difference(pootlefile_set):
self.pofiles[filename] = pootlefile.pootlefile(self, filename)
# remove any files that have been deleted since initialization
for filename in pootlefile_set.difference(filename_set):
del self.pofiles[filename]
def getuploadpath(self, dirname, localfilename):
"""gets the path of a translation file being uploaded securely, creating directories as neccessary"""
if os.path.isabs(dirname) or dirname.startswith("."):
raise ValueError("invalid/insecure file path: %s" % dirname)
if os.path.basename(localfilename) != localfilename or localfilename.startswith("."):
raise ValueError("invalid/insecure file name: %s" % localfilename)
if self.filestyle == "gnu":
if not self.potree.languagematch(self.languagecode, localfilename[:-len("."+self.fileext)]):
raise ValueError("invalid GNU-style file name %s: must match '%s.%s' or '%s[_-][A-Z]{2,3}.%s'" % (localfilename, self.fileext, self.languagecode, self.languagecode, self.fileext))
dircheck = self.podir
for part in dirname.split(os.sep):
dircheck = os.path.join(dircheck, part)
if dircheck and not os.path.isdir(dircheck):
os.mkdir(dircheck)
return os.path.join(self.podir, dirname, localfilename)
def uploadfile(self, session, dirname, filename, contents, overwrite=False):
"""uploads an individual file"""
pathname = self.getuploadpath(dirname, filename)
for extention in ["xliff", "xlf", "xlff"]:
if filename.endswith(extention):
pofilename = filename[:-len(os.extsep+extention)] + os.extsep + self.fileext
popathname = self.getuploadpath(dirname, pofilename)
break
else:
pofilename = filename
popathname = pathname
rights = self.getrights(session)
if os.path.exists(popathname) and not overwrite:
origpofile = self.getpofile(os.path.join(dirname, pofilename))
newfileclass = factory.getclass(pathname)
infile = cStringIO.StringIO(contents)
newfile = newfileclass.parsefile(infile)
if "admin" in rights:
origpofile.mergefile(newfile, session.username)
elif "translate" in rights:
origpofile.mergefile(newfile, session.username, allownewstrings=False)
elif "suggest" in rights:
origpofile.mergefile(newfile, session.username, suggestions=True)
else:
raise RightsError(session.localize("You do not have rights to upload files here"))
else:
if overwrite and not ("admin" in rights or "overwrite" in rights):
raise RightsError(session.localize("You do not have rights to overwrite files here"))
elif not os.path.exists(popathname) and not ("admin" in rights or "overwrite" in rights):
raise RightsError(session.localize("You do not have rights to upload new files here"))
outfile = open(popathname, "wb")
outfile.write(contents)
outfile.close()
self.scanpofiles()
def updatepofile(self, session, dirname, pofilename):
"""updates an individual PO file from version control"""
if "admin" not in self.getrights(session):
raise RightsError(session.localize("You do not have rights to update files here"))
pathname = self.getuploadpath(dirname, pofilename)
# read from version control
if os.path.exists(pathname):
popath = os.path.join(dirname, pofilename)
currentpofile = self.getpofile(popath)
# reading BASE version of file
origcontents = versioncontrol.getcleanfile(pathname, "BASE")
origpofile = pootlefile.pootlefile(self, popath)
originfile = cStringIO.StringIO(origcontents)
origpofile.parse(originfile)
# matching current file with BASE version
matches = origpofile.matchitems(currentpofile, uselocations=False)
# TODO: add some locking here...
# reading new version of file
versioncontrol.updatefile(pathname)
newpofile = pootlefile.pootlefile(self, popath)
newpofile.pofreshen()
if not hasattr(newpofile, "sourceindex"):
newpofile.makeindex()
newmatches = []
# sorting through old matches
for origpo, localpo in matches:
# we need to find the corresponding newpo to see what to merge
if localpo is None:
continue
if origpo is None:
# if it wasn't in the original, then use the addition for searching
origpo = localpo
else:
origmsgstr = origpo.target
localmsgstr = localpo.target
if origmsgstr == localmsgstr:
continue
foundsource = False
# First try to find a match on location
for location in origpo.getlocations():
if location in newpofile.locationindex:
newpo = newpofile.locationindex[location]
if newpo is not None and newpo.source == localpo.source:
foundsource = True
newmatches.append((newpo, localpo))
continue
if not foundsource:
source = origpo.source
if source in newpofile.sourceindex:
newpo = newpofile.sourceindex[source]
newmatches.append((newpo, localpo))
else:
newmatches.append((None, localpo))
# finding new matches
for newpo, localpo in newmatches:
if newpo is None:
# TODO: include localpo as obsolete
continue
if localpo is None:
continue
newpofile.mergeitem(newpo, localpo, "versionmerge")
# saving
newpofile.savepofile()
newpofile.reset_statistics()
self.pofiles[pofilename] = newpofile
# recalculate everything
newpofile.readpofile()
else:
versioncontrol.updatefile(pathname)
self.scanpofiles()
def commitpofile(self, session, dirname, pofilename):
"""commits an individual PO file to version control"""
if "commit" not in self.getrights(session):
raise RightsError(session.localize("You do not have rights to commit files here"))
pathname = self.getuploadpath(dirname, pofilename)
stats = self.getquickstats([os.path.join(dirname, pofilename)])
statsstring = "%d of %d messages translated (%d fuzzy)." % \
(stats["translated"], stats["total"], stats["fuzzy"])
versioncontrol.commitfile(pathname, message="Commit from %s by user %s. %s" %
(session.server.instance.title, session.username, statsstring),
author="%s <%s>" % (session.prefs.name, session.prefs.email))
def converttemplates(self, session):
"""creates PO files from the templates"""
projectdir = os.path.join(self.potree.podirectory, self.projectcode)
if not os.path.exists(projectdir):
os.mkdir(projectdir)
templatesdir = os.path.join(projectdir, "templates")
if not os.path.exists(templatesdir):
templatesdir = os.path.join(projectdir, "pot")
if not os.path.exists(templatesdir):
templatesdir = projectdir
if self.potree.isgnustyle(self.projectcode):
self.filestyle = "gnu"
else:
self.filestyle = "std"
templates = self.potree.gettemplates(self.projectcode)
if self.filestyle == "gnu":
self.podir = projectdir
if not templates:
raise NotImplementedError("Cannot create GNU-style translation project without templates")
else:
self.podir = os.path.join(projectdir, self.languagecode)
if not os.path.exists(self.podir):
os.mkdir(self.podir)
for potfilename in templates:
inputfile = open(os.path.join(templatesdir, potfilename), "rb")
outputfile = cStringIO.StringIO()
if self.filestyle == "gnu":
pofilename = self.languagecode + os.extsep + "po"
else:
pofilename = potfilename[:-len(os.extsep+"pot")] + os.extsep + "po"
pofilename = os.path.basename(pofilename)
origpofilename = os.path.join(self.podir, pofilename)
if os.path.exists(origpofilename):
origpofile = open(origpofilename)
else:
origpofile = None
pot2po.convertpot(inputfile, outputfile, origpofile)
outfile = open(origpofilename, "wb")
outfile.write(outputfile.getvalue())
outfile.close()
self.scanpofiles()
def filtererrorhandler(self, functionname, str1, str2, e):
print "error in filter %s: %r, %r, %s" % (functionname, str1, str2, e)
return False
def getarchive(self, pofilenames):
"""returns an archive of the given filenames"""
try:
# using zip command line is fast
from tempfile import mkstemp
# The temporary file below is opened and immediately closed for security reasons
fd, tempzipfile = mkstemp(prefix='pootle', suffix='.zip')
os.close(fd)
os.system("cd %s ; zip -r - %s > %s" % (self.podir, " ".join(pofilenames), tempzipfile))
filedata = open(tempzipfile, "r").read()
if filedata:
return filedata
finally:
if os.path.exists(tempzipfile):
os.remove(tempzipfile)
# but if it doesn't work, we can do it from python
import zipfile
archivecontents = cStringIO.StringIO()
archive = zipfile.ZipFile(archivecontents, 'w', zipfile.ZIP_DEFLATED)
for pofilename in pofilenames:
pofile = self.getpofile(pofilename)
archive.write(pofile.filename, pofilename)
archive.close()
return archivecontents.getvalue()
def uploadarchive(self, session, dirname, archivecontents):
"""uploads the files inside the archive"""
# Bug #402
#try:
# from tempfile import mktemp
# tempzipfile = mkstemp()
# using zip command line is fast
# os.system("cd %s ; zip -r - %s > %s" % (self.podir, " ".join(pofilenames), tempzipfile))
# return open(tempzipfile, "r").read()
#finally:
# if os.path.exists(tempzipfile):
# os.remove(tempzipfile)
# but if it doesn't work, we can do it from python
import zipfile
archivefile = cStringIO.StringIO(archivecontents)
archive = zipfile.ZipFile(archivefile, 'r')
# TODO: find a better way to return errors...
for filename in archive.namelist():
if not filename.endswith(os.extsep + self.fileext):
print "error adding %s: not a %s file" % (filename, os.extsep + self.fileext)
continue
contents = archive.read(filename)
subdirname, pofilename = os.path.dirname(filename), os.path.basename(filename)
try:
# TODO: use zipfile info to set the time and date of the file
self.uploadfile(session, os.path.join(dirname, subdirname), pofilename, contents)
except ValueError, e:
print "error adding %s" % filename, e
continue
archive.close()
def ootemplate(self):
"""Tests whether this project has an OpenOffice.org template SDF file in
the templates directory."""
projectdir = os.path.join(self.potree.podirectory, self.projectcode)
templatefilename = os.path.join(projectdir, "templates", "en-US.sdf")
if os.path.exists(templatefilename):
return templatefilename
else:
return None
def getoo(self):
"""Returns an OpenOffice.org gsi file"""
#TODO: implement caching
templateoo = self.ootemplate()
if templateoo is None:
return
outputoo = os.path.join(self.podir, self.languagecode + ".sdf")
inputdir = os.path.join(self.potree.podirectory, self.projectcode, self.languagecode)
po2oo.main(["-i%s"%inputdir, "-t%s"%templateoo, "-o%s"%outputoo, "-l%s"%self.languagecode, "--progress=none"])
return file(os.path.join(self.podir, self.languagecode + ".sdf"), "r").read()
def browsefiles(self, dirfilter=None, depth=None, maxdepth=None, includedirs=False, includefiles=True):
"""gets a list of pofilenames, optionally filtering with the parent directory"""
if dirfilter is None:
pofilenames = self.pofilenames
else:
if not dirfilter.endswith(os.path.sep) and not dirfilter.endswith(os.extsep + self.fileext):
dirfilter += os.path.sep
pofilenames = [pofilename for pofilename in self.pofilenames if pofilename.startswith(dirfilter)]
if includedirs:
podirs = {}
for pofilename in pofilenames:
dirname = os.path.dirname(pofilename)
if not dirname:
continue
podirs[dirname] = True
while dirname:
dirname = os.path.dirname(dirname)
if dirname:
podirs[dirname] = True
podirs = podirs.keys()
else:
podirs = []
if not includefiles:
pofilenames = []
if maxdepth is not None:
pofilenames = [pofilename for pofilename in pofilenames if pofilename.count(os.path.sep) <= maxdepth]
podirs = [podir for podir in podirs if podir.count(os.path.sep) <= maxdepth]
if depth is not None:
pofilenames = [pofilename for pofilename in pofilenames if pofilename.count(os.path.sep) == depth]
podirs = [podir for podir in podirs if podir.count(os.path.sep) == depth]
return pofilenames + podirs
def iterpofilenames(self, lastpofilename=None, includelast=False):
"""iterates through the pofilenames starting after the given pofilename"""
if not lastpofilename:
index = 0
else:
index = self.pofilenames.index(lastpofilename)
if not includelast:
index += 1
while index < len(self.pofilenames):
yield self.pofilenames[index]
index += 1
def getindexer(self):
"""get an indexing object for this project
Since we do not want to keep the indexing databases open for the lifetime of
the TranslationProject (it is cached!), it may NOT be part of the Project object,
but should be used via a short living local variable.
"""
indexdir = os.path.join(self.podir, self.index_directory)
index = indexer.get_indexer(indexdir)
index.set_field_analyzers({
"pofilename": index.ANALYZER_EXACT,
"itemno": index.ANALYZER_EXACT,
"pomtime": index.ANALYZER_EXACT})
return index
def initindex(self):
"""initializes the search index"""
if not indexer.HAVE_INDEXER:
return
pofilenames = self.pofiles.keys()
pofilenames.sort()
for pofilename in pofilenames:
self.updateindex(pofilename, optimize=False)
def updateindex(self, pofilename, items=None, optimize=True):
"""updates the index with the contents of pofilename (limit to items if given)
There are three reasons for calling this function:
(A) creating a new instance of "TranslationProject" (see "initindex")
-> check if the index is up-to-date / rebuild the index if necessary
(B) translating a unit via the web interface
-> (re)index only the specified unit(s)
The argument "items" should be None for (A).
known problems:
1) This function should get called, when the po file changes externally.
The function "pofreshen" in pootlefile.py would be the natural place
for this. But this causes circular calls between the current (r7514)
statistics code and "updateindex" leading to indexing database lock
issues.
WARNING: You have to stop the pootle server before manually changing
po files, if you want to keep the index database in sync.
@param pofilename: absolute filename of the po file
@type pofilename: str
@param items: list of unit numbers within the po file OR None (=rebuild all)
@type items: [int]
@param optimize: should the indexing database be optimized afterwards
@type optimize: bool
"""
if not indexer.HAVE_INDEXER:
return
index = self.getindexer()
pofile = self.pofiles[pofilename]
# check if the pomtime in the index == the latest pomtime
try:
pomtime = statistics.getmodtime(pofile.filename)
pofilenamequery = index.make_query([("pofilename", pofilename)], True)
pomtimequery = index.make_query([("pomtime", str(pomtime))], True)
gooditemsquery = index.make_query([pofilenamequery, pomtimequery], True)
gooditemsnum = index.get_query_result(gooditemsquery).get_matches_count()
# if there is at least one up-to-date indexing item, then the po file
# was not changed externally -> no need to update the database
if (gooditemsnum > 0) and (not items):
# nothing to be done
return
elif items:
# Update only specific items - usually single translation via the web
# interface. All other items should still be up-to-date (even with an
# older pomtime).
print "updating", self.languagecode, "index for", pofilename, "items", items
# delete the relevant items from the database
itemsquery = index.make_query([("itemno", str(itemno)) for itemno in items], False)
index.delete_doc([pofilenamequery, itemsquery])
else:
# (items is None)
# The po file is not indexed - or it was changed externally (see
# "pofreshen" in pootlefile.py).
print "updating", self.projectcode, self.languagecode, "index for", pofilename
# delete all items of this file
index.delete_doc({"pofilename": pofilename})
pofile.pofreshen()
if items is None:
# rebuild the whole index
items = range(pofile.statistics.getitemslen())
addlist = []
for itemno in items:
unit = pofile.getitem(itemno)
doc = {"pofilename": pofilename, "pomtime": str(pomtime), "itemno": str(itemno)}
if unit.hasplural():
orig = "\n".join(unit.source.strings)
trans = "\n".join(unit.target.strings)
else:
orig = unit.source
trans = unit.target
doc["msgid"] = orig
doc["msgstr"] = trans
addlist.append(doc)
if addlist:
index.begin_transaction()
try:
for add_item in addlist:
index.index_document(add_item)
finally:
index.commit_transaction()
index.flush(optimize=optimize)
except (base.ParseError, IOError, OSError):
index.delete_doc({"pofilename": pofilename})
print "Not indexing %s, since it is corrupt" % (pofilename,)
def matchessearch(self, pofilename, search):
"""returns whether any items in the pofilename match the search (based on collected stats etc)"""
# wrong file location in a "dirfilter" search?
if search.dirfilter is not None and not pofilename.startswith(search.dirfilter):
return False
# search.assignedto == [None] means assigned to nobody
if search.assignedto or search.assignedaction:
if search.assignedto == [None]:
assigns = self.pofiles[pofilename].getassigns().getunassigned(search.assignedaction)
else:
assigns = self.pofiles[pofilename].getassigns().getassigns()
if search.assignedto is not None:
if search.assignedto not in assigns:
return False
assigns = assigns[search.assignedto]
else:
assigns = reduce(lambda x, y: x+y, [userassigns.keys() for userassigns in assigns.values()], [])
if search.assignedaction is not None:
if search.assignedaction not in assigns:
return False
if search.matchnames:
postats = self.getpostats(pofilename)
matches = False
for name in search.matchnames:
if postats.get(name):
matches = True
if not matches:
return False
return True
def indexsearch(self, search, returnfields):
"""returns the results from searching the index with the given search"""
if not indexer.HAVE_INDEXER:
return False
index = self.getindexer()
searchparts = []
if search.searchtext:
textquery = index.make_query([("msgid", search.searchtext), ("msgstr", search.searchtext)], False)
searchparts.append(textquery)
if search.dirfilter:
pofilenames = self.browsefiles(dirfilter=search.dirfilter)
filequery = index.make_query([("pofilename", pofilename) for pofilename in pofilenames], False)
searchparts.append(filequery)
# TODO: add other search items
limitedquery = index.make_query(searchparts, True)
return index.search(limitedquery, returnfields)
def searchpofilenames(self, lastpofilename, search, includelast=False):
"""find the next pofilename that has items matching the given search"""
if lastpofilename and not lastpofilename in self.pofiles:
# accessing will autoload this file...
self.pofiles[lastpofilename]
if indexer.HAVE_INDEXER and search.searchtext:
# TODO: move this up a level, use index to manage whole search, so we don't do this twice
hits = self.indexsearch(search, "pofilename")
# there will be only result for the field "pofilename" - so we just
# pick the first
searchpofilenames = dict.fromkeys([hit["pofilename"][0] for hit in hits])
else:
searchpofilenames = None
for pofilename in self.iterpofilenames(lastpofilename, includelast):
if searchpofilenames is not None:
if pofilename not in searchpofilenames:
continue
if self.matchessearch(pofilename, search):
yield pofilename
def searchpoitems(self, pofilename, item, search):
"""finds the next item matching the given search"""
if search.searchtext:
grepfilter = pogrep.GrepFilter(search.searchtext, None, ignorecase=True)
for pofilename in self.searchpofilenames(pofilename, search, includelast=True):
pofile = self.getpofile(pofilename)
if indexer.HAVE_INDEXER and (search.searchtext or search.matchnames):
filesearch = search.copy()
filesearch.dirfilter = pofilename
hits = self.indexsearch(filesearch, "itemno")
# there will be only result for the field "itemno" - so we just
# pick the first
items = [int(doc["itemno"][0]) for doc in hits]
items = [searchitem for searchitem in items if searchitem > item]
items.sort()
notextsearch = search.copy()
notextsearch.searchtext = None
matchitems = list(pofile.iteritems(notextsearch, item))
else:
items = pofile.iteritems(search, item)
matchitems = items
for item in items:
if items != matchitems:
if item not in matchitems:
continue
# TODO: move this to iteritems
if search.searchtext:
unit = pofile.getitem(item)
if grepfilter.filterunit(unit):
yield pofilename, item
else:
yield pofilename, item
item = None
def reassignpoitems(self, session, search, assignto, action):
"""reassign all the items matching the search to the assignto user(s) evenly, with the given action"""
# remove all assignments for the given action
self.unassignpoitems(session, search, None, action)
assigncount = self.assignpoitems(session, search, assignto, action)
return assigncount
def assignpoitems(self, session, search, assignto, action):
"""assign all the items matching the search to the assignto user(s) evenly, with the given action"""
if not "assign" in self.getrights(session):
raise RightsError(session.localize("You do not have rights to alter assignments here"))
if search.searchtext:
grepfilter = pogrep.GrepFilter(search.searchtext, None, ignorecase=True)
if not isinstance(assignto, list):
assignto = [assignto]
usercount = len(assignto)
assigncount = 0
if not usercount:
return assigncount
pofilenames = [pofilename for pofilename in self.searchpofilenames(None, search, includelast=True)]
wordcounts = [(pofilename, self.getpofile(pofilename).statistics.getquickstats()['totalsourcewords']) for pofilename in pofilenames]
totalwordcount = sum([wordcount for pofilename, wordcount in wordcounts])
wordsperuser = totalwordcount / usercount
print "assigning", totalwordcount, "words to", usercount, "user(s)", wordsperuser, "words per user"
usernum = 0
userwords = 0
for pofilename, wordcount in wordcounts:
pofile = self.getpofile(pofilename)
sourcewordcount = pofile.statistics.getunitstats()['sourcewordcount']
for item in pofile.iteritems(search, None):
# TODO: move this to iteritems
if search.searchtext:
validitem = False
unit = pofile.getitem(item)
if grepfilter.filterunit(unit):
validitem = True
if not validitem:
continue
itemwordcount = sourcewordcount[item]
#itemwordcount = statsdb.wordcount(str(pofile.getitem(item).source))
if userwords + itemwordcount > wordsperuser:
usernum = min(usernum+1, len(assignto)-1)
userwords = 0
userwords += itemwordcount
pofile.getassigns().assignto(item, assignto[usernum], action)
assigncount += 1
return assigncount
def unassignpoitems(self, session, search, assignedto, action=None):
"""unassigns all the items matching the search to the assignedto user"""
if not "assign" in self.getrights(session):
raise RightsError(session.localize("You do not have rights to alter assignments here"))
if search.searchtext:
grepfilter = pogrep.GrepFilter(search.searchtext, None, ignorecase=True)
assigncount = 0
for pofilename in self.searchpofilenames(None, search, includelast=True):
pofile = self.getpofile(pofilename)
for item in pofile.iteritems(search, None):
# TODO: move this to iteritems
if search.searchtext:
unit = pofile.getitem(item)
if grepfilter.filterunit(unit):
pofile.getassigns().unassign(item, assignedto, action)
assigncount += 1
else:
pofile.getassigns().unassign(item, assignedto, action)
assigncount += 1
return assigncount
def updatequickstats(self, pofilename, translatedwords, translated, fuzzywords, fuzzy, totalwords, total, save=True):
"""updates the quick stats on the given file"""
self.quickstats[pofilename] = (translatedwords, translated, fuzzywords, fuzzy, totalwords, total)
if save:
self.savequickstats()
def savequickstats(self):
"""saves the quickstats"""
self.quickstatsfilename = os.path.join(self.podir, "pootle-%s-%s.stats" % (self.projectcode, self.languagecode))
quickstatsfile = open(self.quickstatsfilename, "w")
sortedquickstats = self.quickstats.items()
sortedquickstats.sort()
for pofilename, (translatedwords, translated, fuzzywords, fuzzy, totalwords, total) in sortedquickstats:
quickstatsfile.write("%s, %d, %d, %d, %d, %d, %d\n" % \
(pofilename, translatedwords, translated, fuzzywords, fuzzy, totalwords, total))
quickstatsfile.close()
def readquickstats(self):
"""reads the quickstats from disk"""
self.quickstats = {}
self.quickstatsfilename = os.path.join(self.podir, "pootle-%s-%s.stats" % (self.projectcode, self.languagecode))
if os.path.exists(self.quickstatsfilename):
quickstatsfile = open(self.quickstatsfilename, "r")
for line in quickstatsfile:
items = line.split(",")
if len(items) != 7:
#Must be an old format style without the fuzzy stats
self.quickstats = self.getquickstats()
self.savequickstats()
break
else:
pofilename, translatedwords, translated, fuzzywords, fuzzy, totalwords, total = items
self.quickstats[pofilename] = tuple([int(a.strip()) for a in \
translatedwords, translated, fuzzywords, fuzzy, totalwords, total])
def getquickstats(self, pofilenames=None):
"""Gets translated and total stats and wordcounts without doing calculations returning dictionary."""
if pofilenames is None:
pofilenames = self.pofilenames
alltranslatedwords, alltranslated, allfuzzywords, allfuzzy, alltotalwords, alltotal = 0, 0, 0, 0, 0, 0
slowfiles = []
for pofilename in pofilenames:
if pofilename not in self.quickstats:
slowfiles.append(pofilename)
continue
translatedwords, translated, fuzzywords, fuzzy, totalwords, total = self.quickstats[pofilename]
alltranslatedwords += translatedwords
alltranslated += translated
allfuzzywords += fuzzywords
allfuzzy += fuzzy
alltotalwords += totalwords
alltotal += total
for pofilename in slowfiles:
self.pofiles[pofilename].statistics.updatequickstats(save=False)
translatedwords, translated, fuzzywords, fuzzy, totalwords, total = self.quickstats[pofilename]
alltranslatedwords += translatedwords
alltranslated += translated
allfuzzywords += fuzzywords
allfuzzy += fuzzy
alltotalwords += totalwords
alltotal += total
if slowfiles:
self.savequickstats()
return {"translatedsourcewords": alltranslatedwords, "translated": alltranslated,
"fuzzysourcewords": allfuzzywords, "fuzzy": allfuzzy,
"totalsourcewords": alltotalwords, "total": alltotal}
def combinestats(self, pofilenames=None):
"""combines translation statistics for the given po files (or all if None given)"""
if pofilenames is None:
pofilenames = self.pofilenames
pofilenames = [pofilename for pofilename in pofilenames
if pofilename != None and not os.path.isdir(pofilename)]
total_stats = self.combine_totals(pofilenames)
total_stats['units'] = self.combine_unit_stats(pofilenames)
total_stats['assign'] = self.combineassignstats(pofilenames)
return total_stats
def combine_totals(self, pofilenames):
totalstats = {}
for pofilename in pofilenames:
pototals = self.getpototals(pofilename)
for name, items in pototals.iteritems():
totalstats[name] = totalstats.get(name, 0) + pototals[name]
return totalstats
def combine_unit_stats(self, pofilenames):
unit_stats = {}
for pofilename in pofilenames:
postats = self.getpostats(pofilename)
for name, items in postats.iteritems():
unit_stats.setdefault(name, []).extend([(pofilename, item) for item in items])
return unit_stats
def combineassignstats(self, pofilenames=None, action=None):
"""combines assign statistics for the given po files (or all if None given)"""
assign_stats = {}
for pofilename in pofilenames:
assignstats = self.getassignstats(pofilename, action)
for name, items in assignstats.iteritems():
assign_stats.setdefault(name, []).extend([(pofilename, item) for item in items])
return assign_stats
def countwords(self, stats):
"""counts the number of words in the items represented by the stats list"""
wordcount = 0
for pofilename, item in stats:
pofile = self.pofiles[pofilename]
if 0 <= item < len(pofile.statistics.getunitstats()['sourcewordcount']):
wordcount += pofile.statistics.getunitstats()['sourcewordcount'][item]
print "projects::countwords()"
return wordcount
def getpomtime(self):
"""returns the modification time of the last modified file in the project"""
return max([pofile.pomtime for pofile in self.pofiles.values()])
pomtime = property(getpomtime)
def track(self, pofilename, item, message):
"""sends a track message to the pofile"""
self.pofiles[pofilename].track(item, message)
def gettracks(self, pofilenames=None):
"""calculates translation statistics for the given po files (or all if None given)"""
alltracks = []
if pofilenames is None:
pofilenames = self.pofilenames
for pofilename in pofilenames:
if not pofilename or os.path.isdir(pofilename):
continue
tracker = self.pofiles[pofilename].tracker
items = tracker.keys()
items.sort()
for item in items:
alltracks.append("%s item %d: %s" % (pofilename, item, tracker[item]))
return alltracks
def getpostats(self, pofilename):
"""calculates translation statistics for the given po file"""
return self.pofiles[pofilename].statistics.getstats()
def getpototals(self, pofilename):
"""calculates translation statistics for the given po file"""
return self.pofiles[pofilename].statistics.getquickstats()
def getassignstats(self, pofilename, action=None):
"""calculates translation statistics for the given po file (can filter by action if given)"""
polen = self.getpototals(pofilename).get("total", 0)
# Temporary code to avoid traceback. Was:
# polen = len(self.getpostats(pofilename)["total"])
assigns = self.pofiles[pofilename].getassigns().getassigns()
assignstats = {}
for username, userassigns in assigns.iteritems():
allitems = []
for assignaction, items in userassigns.iteritems():
if action is None or assignaction == action:
allitems += [item for item in items if 0 <= item < polen and item not in allitems]
if allitems:
assignstats[username] = allitems
return assignstats
def getpofile(self, pofilename, freshen=True):
"""parses the file into a pofile object and stores in self.pofiles"""
pofile = self.pofiles[pofilename]
if freshen:
pofile.pofreshen()
return pofile
def getpofilelen(self, pofilename):
"""returns number of items in the given pofilename"""
pofile = self.getpofile(pofilename)
return pofile.statistics.getitemslen()
def getitems(self, pofilename, itemstart, itemstop):
"""returns a set of items from the pofile, converted to original and translation strings"""
pofile = self.getpofile(pofilename)
units = [pofile.units[index] for index in pofile.statistics.getstats()["total"][max(itemstart,0):itemstop]]
return units
def updatetranslation(self, pofilename, item, newvalues, session):
"""updates a translation with a new value..."""
if "translate" not in self.getrights(session):
raise RightsError(session.localize("You do not have rights to change translations here"))
pofile = self.pofiles[pofilename]
pofile.track(item, "edited by %s" % session.username)
languageprefs = getattr(session.instance.languages, self.languagecode, None)
pofile.updateunit(item, newvalues, session.prefs, languageprefs)
self.updateindex(pofilename, [item])
def suggesttranslation(self, pofilename, item, trans, session):
"""stores a new suggestion for a translation..."""
if "suggest" not in self.getrights(session):
raise RightsError(session.localize("You do not have rights to suggest changes here"))
pofile = self.getpofile(pofilename)
pofile.track(item, "suggestion made by %s" % session.username)
pofile.addsuggestion(item, trans, session.username)
def getsuggestions(self, pofile, item):
"""find all the suggestions submitted for the given (pofile or pofilename) and item"""
if isinstance(pofile, (str, unicode)):
pofilename = pofile
pofile = self.getpofile(pofilename)
suggestpos = pofile.getsuggestions(item)
return suggestpos
def acceptsuggestion(self, pofile, item, suggitem, newtrans, session):
"""accepts the suggestion into the main pofile"""
if not "review" in self.getrights(session):
raise RightsError(session.localize("You do not have rights to review suggestions here"))
if isinstance(pofile, (str, unicode)):
pofilename = pofile
pofile = self.getpofile(pofilename)
pofile.track(item, "suggestion by %s accepted by %s" % (self.getsuggester(pofile, item, suggitem), session.username))
pofile.deletesuggestion(item, suggitem)
self.updatetranslation(pofilename, item, {"target": newtrans, "fuzzy": False}, session)
def getsuggester(self, pofile, item, suggitem):
"""returns who suggested the given item's suggitem if recorded, else None"""
if isinstance(pofile, (str, unicode)):
pofilename = pofile
pofile = self.getpofile(pofilename)
return pofile.getsuggester(item, suggitem)
def rejectsuggestion(self, pofile, item, suggitem, newtrans, session):
"""rejects the suggestion and removes it from the pending file"""
if not "review" in self.getrights(session):
raise RightsError(session.localize("You do not have rights to review suggestions here"))
if isinstance(pofile, (str, unicode)):
pofilename = pofile
pofile = self.getpofile(pofilename)
pofile.track(item, "suggestion by %s rejected by %s" % (self.getsuggester(pofile, item, suggitem), session.username))
pofile.deletesuggestion(item, suggitem)
def gettmsuggestions(self, pofile, item):
"""find all the TM suggestions for the given (pofile or pofilename) and item"""
if isinstance(pofile, (str, unicode)):
pofilename = pofile
pofile = self.getpofile(pofilename)
tmsuggestpos = pofile.gettmsuggestions(item)
return tmsuggestpos
def isterminologyproject(self):
"""returns whether this project is the main terminology project for a
language. Currently it is indicated by the project code 'terminology'"""
return self.projectcode == "terminology"
def gettermbase(self):
"""returns this project's terminology store"""
if self.isterminologyproject():
if len(self.pofiles) > 0:
for termfile in self.pofiles.values():
termfile.pofreshen()
return self
else:
termfilename = "pootle-terminology."+self.fileext
if termfilename in self.pofiles:
termfile = self.getpofile(termfilename, freshen=True)
return termfile
return None
def gettermmatcher(self):
"""returns the terminology matcher"""
termbase = self.gettermbase()
if termbase:
newmtime = termbase.pomtime
if newmtime != self.termmatchermtime:
self.termmatchermtime = newmtime
if self.isterminologyproject():
self.termmatcher = match.terminologymatcher(self.pofiles.values())
else:
self.termmatcher = match.terminologymatcher(termbase)
elif not self.isterminologyproject() and self.potree.hasproject(self.languagecode, "terminology"):
termproject = self.potree.getproject(self.languagecode, "terminology")
self.termmatcher = termproject.gettermmatcher()
self.termmatchermtime = termproject.termmatchermtime
else:
self.termmatcher = None
self.termmatchermtime = None
return self.termmatcher
def getterminology(self, session, pofile, item):
"""find all the terminology for the given (pofile or pofilename) and item"""
try:
termmatcher = self.gettermmatcher()
if not termmatcher:
return []
if isinstance(pofile, (str, unicode)):
pofilename = pofile
pofile = self.getpofile(pofilename)
return termmatcher.matches(pofile.getitem(item).source)
except Exception, e:
session.server.errorhandler.logerror(traceback.format_exc())
return []
def savepofile(self, pofilename):
"""saves changes to disk"""
pofile = self.getpofile(pofilename)
pofile.savepofile()
def getoutput(self, pofilename):
"""returns pofile source"""
pofile = self.getpofile(pofilename)
return pofile.getoutput()
def convert(self, pofilename, destformat):
"""converts the pofile to the given format, returning (etag_if_filepath, filepath_or_contents)"""
pofile = self.getpofile(pofilename, freshen=False)
destfilename = pofile.filename[:-len(self.fileext)] + destformat
destmtime = statistics.getmodtime(destfilename)
pomtime = statistics.getmodtime(pofile.filename)
if pomtime and destmtime == pomtime:
try:
return pomtime, destfilename
except Exception, e:
print "error reading cached converted file %s: %s" % (destfilename, e)
pofile.pofreshen()
converters = {"csv": po2csv.po2csv, "xlf": po2xliff.po2xliff, "po": xliff2po.xliff2po, "ts": po2ts.po2ts, "mo": pocompile.POCompile}
converterclass = converters.get(destformat, None)
if converterclass is None:
raise ValueError("No converter available for %s" % destfilename)
contents = converterclass().convertstore(pofile)
if not isinstance(contents, basestring):
contents = str(contents)
try:
destfile = open(destfilename, "w")
destfile.write(contents)
destfile.close()
currenttime, modtime = time.time(), pofile.pomtime
os.utime(destfilename, (currenttime, modtime))
return modtime, destfilename
except Exception, e:
print "error caching converted file %s: %s" % (destfilename, e)
return False, contents
def gettext(self, message):
"""uses the project as a live translator for the given message"""
for pofilename, pofile in self.pofiles.iteritems():
if pofile.pomtime != statistics.getmodtime(pofile.filename):
pofile.readpofile()
pofile.makeindex()
elif not hasattr(pofile, "sourceindex"):
pofile.makeindex()
unit = pofile.sourceindex.get(message, None)
if not unit or not unit.istranslated():
continue
tmsg = unit.target
if tmsg is not None:
return tmsg
return message
def ugettext(self, message):
"""gets the translation of the message by searching through all the pofiles (unicode version)"""
for pofilename, pofile in self.pofiles.iteritems():
try:
if pofile.pofreshen() or not hasattr(pofile, "sourceindex"):
pofile.makeindex()
unit = pofile.sourceindex.get(message, None)
if not unit or not unit.istranslated():
continue
tmsg = unit.target
if tmsg is not None:
if isinstance(tmsg, unicode):
return tmsg
else:
return unicode(tmsg, pofile.encoding)
except Exception, e:
print "error reading translation from pofile %s: %s" % (pofilename, e)
return unicode(message)
def ungettext(self, singular, plural, n):
"""gets the plural translation of the message by searching through all the pofiles (unicode version)"""
for pofilename, pofile in self.pofiles.iteritems():
try:
if pofile.pomtime != statistics.getmodtime(pofile.filename):
pofile.readpofile()
pofile.makeindex()
elif not hasattr(pofile, "sourceindex"):
pofile.makeindex()
nplural, pluralequation = pofile.getheaderplural()
if pluralequation:
pluralfn = gettext.c2py(pluralequation)
unit = pofile.sourceindex.get(singular, None)
if not unit or not unit.istranslated():
continue
tmsg = unit.target.strings[pluralfn(n)]
if tmsg is not None:
if isinstance(tmsg, unicode):
return tmsg
else:
return unicode(tmsg, pofile.encoding)
except Exception, e:
print "error reading translation from pofile %s: %s" % (pofilename, e)
if n == 1:
return unicode(singular)
else:
return unicode(plural)
def hascreatemofiles(self, projectcode):
"""returns whether the project has createmofile set"""
return self.potree.getprojectcreatemofiles(projectcode) == 1
class DummyProject(TranslationProject):
"""a project that is just being used for handling pootlefiles"""
def __init__(self, podir, checker=None, projectcode=None, languagecode=None):
"""initializes the project with the given podir"""
self.podir = podir
if checker is None:
self.checker = checks.TeeChecker()
else:
self.checker = checker
self.projectcode = projectcode
self.languagecode = languagecode
self.readquickstats()
def scanpofiles(self):
"""A null operation if potree is not present"""
pass
def readquickstats(self):
"""dummy statistics are empty"""
self.quickstats = {}
def savequickstats(self):
"""saves quickstats if possible"""
pass
class DummyStatsProject(DummyProject):
"""a project that is just being used for refresh of statistics"""
def __init__(self, podir, checker, projectcode=None, languagecode=None):
"""initializes the project with the given podir"""
DummyProject.__init__(self, podir, checker, projectcode, languagecode)
def readquickstats(self):
"""reads statistics from whatever files are available"""
self.quickstats = {}
if self.projectcode is not None and self.languagecode is not None:
TranslationProject.readquickstats(self)
def savequickstats(self):
"""saves quickstats if possible"""
if self.projectcode is not None and self.languagecode is not None:
TranslationProject.savequickstats(self)
class TemplatesProject(TranslationProject):
"""Manages Template files (.pot files) for a project"""
fileext = "pot"
def __init__(self, projectcode, potree):
super(TemplatesProject, self).__init__("templates", projectcode, potree, create=False)
def getrights(self, session=None, username=None, usedefaults=True):
"""gets the rights for the given user (name or session, or not-logged-in if username is None)"""
# internal admin sessions have all rights
# We don't send the usedefaults parameter through, because we don't want users of this method to
# change the default behaviour in a template project. Yes, I know: ignorance and deceit.
rights = super(TemplatesProject, self).getrights(session=session, username=username)
if rights is not None:
rights = [right for right in rights if right not in ["translate", "suggest", "pocompile"]]
return rights
