Skip to content

Commit

Permalink
Convert Twisted's plugin system to use twisted.python.modules for cod…
Browse files Browse the repository at this point in the history 
…e discovery

This fixes several bugs in the traversal and caching logic of the plugins system, and simplifies it by factoring the code such that the representation of the Python path and import system is now delegated entirely to an API designed to do it.  The other major bug fixed here made plugin modules from a development and system installation of Twisted conflict, even if one path entry clearly "won" from the perspective of sys.path, PYTHONPATH, {{{__import__}}}, et. al.

In addition to the main fix here, several new features were introduced into the {{{twisted.python.filepath}}}, {{{twisted.python.zippath}}} and {{{twisted.python.modules}}} modules.

  * twisted.python.filepath
    * New methods: {{{getStatusChangeTime}}}, {{{getModificationTime}}}, and {{{getAccessTime}}}, to access higher precision timestamps as floats.
    * New exception-handling behavior: {{{FilePath.children}}} will now raise {{{UnlistableError}}}, allowing users to catch non-fatal reasons why a directory might not be listable.  More importantly this allows for ''portable'' error handling between UNIX and Windows.
    * {{{FilePath}}} instances are now usable as dictionary keys.
  * twisted.python.zippath
    * Acquired error-handling similar to {{{FilePath}}}
    * Acquired new methods for inspecting timestamps, both the old deprecated {{{get(m|c|a)time}}} form and the newer form.  This should allow both older and newer code to work with {{{ZipPath}}} instances.  The modification stamp is actually pulled from the zipfile metadata.
  * twisted.python.modules
    * Gratuitous Windows error-handling logic was removed and replaced with a simple {{{except UnlistableError}}} thanks to {{{FilePath}}}'s new features.

Finally, test coverage was improved, and a few gratuitous pyflakes warnings and bits of trailing whitespace were eliminated.

Fixes #1951

Author: glyph

Reviewers: jerub, exarkun


git-svn-id: svn://svn.twistedmatrix.com/svn/Twisted/trunk@19305 bbbe8e31-12d6-0310-92fd-ac37d47ddeeb
  • Loading branch information
@glyph
glyph committed Jan 12, 2007
1 parent d72b6cf commit a8805a0
Show file tree
Hide file tree
Showing 8 changed files with 868 additions and 223 deletions.
211 changes: 85 additions & 126 deletions twisted/plugin.py
View file
Expand Up @@ -11,50 +11,45 @@

from __future__ import generators

import os, errno

from zope.interface import Interface, providedBy

try:
import cPickle as pickle
except ImportError:
import pickle
def _determinePickleModule():
"""
Determine which 'pickle' API module to use.
"""
try:
import cPickle
return cPickle
except ImportError:
import pickle
return pickle

pickle = _determinePickleModule()

from twisted.python.components import getAdapterFactory
from twisted.python.reflect import namedAny
from twisted.python.win32 import ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND
from twisted.python.win32 import ERROR_INVALID_NAME, WindowsError
from twisted.python import log

try:
from os import stat_float_times
from os.path import getmtime as _getmtime
def getmtime(x):
sft = stat_float_times()
stat_float_times(True)
try:
return _getmtime(x)
finally:
stat_float_times(sft)
except:
from os.path import getmtime
from twisted.python.modules import getModule

class IPlugin(Interface):
"""Interface that must be implemented by all plugins.
"""
Interface that must be implemented by all plugins.
Only objects which implement this interface will be considered for
return by C{getPlugins}. To be useful, plugins should also
implement some other application-specific interface.
Only objects which implement this interface will be considered for return
by C{getPlugins}. To be useful, plugins should also implement some other
application-specific interface.
"""

class ITestPlugin(Interface):
"""A plugin for use by the plugin system's unit tests.
"""
A plugin for use by the plugin system's unit tests.
Do not use this.
"""

class ITestPlugin2(Interface):
"""See L{ITestPlugin}.
"""
See L{ITestPlugin}.
"""

class CachedPlugin(object):
Expand Down Expand Up @@ -108,122 +103,86 @@ def fromkeys(keys, value=None):
d[k] = value
return d

_exts = fromkeys(['.py', '.so', '.pyd', '.dll'])

def getCache(module):
topcache = {}
for p in module.__path__:
dropcache = os.path.join(p, "dropin.cache")
"""
Compute all the possible loadable plugins, while loading as few as
possible and hitting the filesystem as little as possible.
@param module: a Python module object. This represents a package to search
for plugins.
@return: a dictionary mapping module names to CachedDropin instances.
"""
allCachesCombined = {}
mod = getModule(module.__name__)
# don't want to walk deep, only immediate children.
lastPath = None
buckets = {}
# Fill buckets with modules by related entry on the given package's
# __path__. There's an abstraction inversion going on here, because this
# information is already represented internally in twisted.python.modules,
# but it's simple enough that I'm willing to live with it. If anyone else
# wants to fix up this iteration so that it's one path segment at a time,
# be my guest. --glyph
for plugmod in mod.iterModules():
fpp = plugmod.filePath.parent()
if fpp not in buckets:
buckets[fpp] = []
bucket = buckets[fpp]
bucket.append(plugmod)
for pseudoPackagePath, bucket in buckets.iteritems():
dropinPath = pseudoPackagePath.child('dropin.cache')
try:
cache = pickle.load(file(dropcache))
lastCached = getmtime(dropcache)
dirtyCache = False
lastCached = dropinPath.getModificationTime()
dropinDotCache = pickle.load(dropinPath.open('rb'))
except:
cache = {}
dropinDotCache = {}
lastCached = 0
dirtyCache = True
try:
dropinNames = os.listdir(p)
except WindowsError, e:
# WindowsError is an OSError subclass, so if not for this clause
# the OSError clause below would be handling these. Windows
# error codes aren't the same as POSIX error codes, so we need
# to handle them differently.

# Under Python 2.5 on Windows, WindowsError has a winerror
# attribute and an errno attribute. The winerror attribute is
# bound to the Windows error code while the errno attribute is
# bound to a translation of that code to a perhaps equivalent
# POSIX error number.

# Under Python 2.4 on Windows, WindowsError only has an errno
# attribute. It is bound to the Windows error code.

# For simplicity of code and to keep the number of paths through
# this suite minimal, we grab the Windows error code under
# either version.

# Furthermore, attempting to use os.listdir on a non-existent
# path in Python 2.4 will result in a Windows error code of
# ERROR_PATH_NOT_FOUND. However, in Python 2.5,
# ERROR_FILE_NOT_FOUND results instead. -exarkun
err = getattr(e, 'winerror', e.errno)
if err in (ERROR_PATH_NOT_FOUND, ERROR_FILE_NOT_FOUND):
continue
elif err == ERROR_INVALID_NAME:
log.msg("Invalid path %r in search path for %s" % (p, module.__name__))
continue
else:
raise
except OSError, ose:
if ose.errno not in (errno.ENOENT, errno.ENOTDIR):
raise
else:
continue
else:
pys = {}
for dropinName in dropinNames:
moduleName, moduleExt = os.path.splitext(dropinName)
if moduleName != '__init__' and moduleExt in _exts:
pyFile = os.path.join(p, dropinName)
try:
pys[moduleName] = getmtime(pyFile)
except:
log.err()

for moduleName, lastChanged in pys.iteritems():
if lastChanged >= lastCached or moduleName not in cache:
dirtyCache = True

needsWrite = False
existingKeys = {}
for pluginModule in bucket:
pluginKey = pluginModule.name.split('.')[-1]
existingKeys[pluginKey] = True
if ((pluginKey not in dropinDotCache) or
(pluginModule.filePath.getModificationTime() >= lastCached)):
needsWrite = True
try:
provider = namedAny(module.__name__ + '.' + moduleName)
provider = pluginModule.load()
except:
# dropinDotCache.pop(pluginKey, None)
log.err()
else:
entry = _generateCacheEntry(provider)
cache[moduleName] = entry
dropinDotCache[pluginKey] = entry
# Make sure that the cache doesn't contain any stale plugins.
for pluginKey in dropinDotCache.keys():
if pluginKey not in existingKeys:
del dropinDotCache[pluginKey]
needsWrite = True
if needsWrite:
try:
dropinPath.setContent(pickle.dumps(dropinDotCache))
except:
log.err()
allCachesCombined.update(dropinDotCache)
return allCachesCombined

for moduleName in cache.keys():
if moduleName not in pys:
dirtyCache = True
del cache[moduleName]

topcache.update(cache)
def getPlugins(interface, package=None):
"""
Retrieve all plugins implementing the given interface beneath the given module.
if dirtyCache:
newCacheData = pickle.dumps(cache, 2)
tmpCacheFile = dropcache + ".new"
try:
stage = 'opening'
f = file(tmpCacheFile, 'wb')
stage = 'writing'
f.write(newCacheData)
stage = 'closing'
f.close()
stage = 'renaming'
os.rename(tmpCacheFile, dropcache)
except (OSError, IOError), e:
# A large number of errors can occur here. There's nothing we
# can really do about any of them, but they are also non-fatal
# (they only slow us down by preventing results from being
# cached). Notify the user of the error, but proceed as if it
# had not occurred.
log.msg("Error %s plugin cache file %r (%r): %r" % (
stage, tmpCacheFile, dropcache, os.strerror(e.errno)))

return topcache

import twisted.plugins
def getPlugins(interface, package=twisted.plugins):
"""Retrieve all plugins implementing the given interface beneath the given module.
@param interface: An interface class. Only plugins which
implement this interface will be returned.
@param interface: An interface class. Only plugins which implement this
interface will be returned.
@param package: A package beneath which plugins are installed. For
most uses, the default value is correct.
@return: An iterator of plugins.
"""
if package is None:
import twisted.plugins as package
allDropins = getCache(package)
for dropin in allDropins.itervalues():
for plugin in dropin.plugins:
Expand All @@ -235,7 +194,7 @@ def getPlugins(interface, package=twisted.plugins):
if adapted is not None:
yield adapted


# Old, backwards compatible name. Don't use this.
getPlugIns = getPlugins

Expand Down

0 comments on commit a8805a0

Please sign in to comment.