Backport bugfixes from the 0.12 release where *pyc and *pyo files are…

… ignored in the scanning process. Likewise, bogus modules such as editor droppings are ignored now as well

CHANGES.txt

@@ -4,8 +4,21 @@ CHANGES
 0.11.2 (unreleased)
 ===================
 
-- Nothing changed yet.
+Bugs fixed
+----------
+
+* Ignore things that look like Python modules and packages but aren't.
+  These are sometimes created by editors, operating systems and
+  network file systems and we don't want to confuse them.
+
+* Ignore .pyc and .pyo files that don't have a matching .py file via
+  ``module_info_from_dotted_name`` if its ``ignore_nonsource``
+  parameter is ``True``.  The default is ``True``.  To revert to the
+  older behavior where .pyc files were honored, pass
+  ``ignore_nonsource=False``.
 
+* Pass along ``exclude_filter`` (and the new ``ignore_nonsource``
+  flag) to ModuleInfo constructor when it calls itself recursively.
 
 0.11.1 (2009-10-07)
 ===================

CREDITS.txt

@@ -13,6 +13,8 @@ CREDITS
 
 * Luciano Ramalho
 
+* Chris McDonough
+
 Thank you
 ---------
 

src/martian/scan.py

@@ -32,14 +32,17 @@ def is_package(path):
 class ModuleInfo(object):
     implements(IModuleInfo)
 
-    def __init__(self, path, dotted_name, exclude_filter=None):
+    def __init__(self, path, dotted_name, exclude_filter=None,
+                 ignore_nonsource=True):
         # Normalize .pyc files to .py
         if path.endswith('c'):
             path = path[:-1]
         self.path = path
         self.dotted_name = dotted_name
+        self.ignore_nonsource = ignore_nonsource
 
         if exclude_filter is None:
+            # this exclude filter receives extensionless filenames
             self.exclude_filter = lambda x: False
         else:
             self.exclude_filter = exclude_filter
@@ -72,11 +75,19 @@ def getSubModuleInfos(self):
         module_infos = []
         seen = []
         for entry in sorted(os.listdir(directory)):
+            # we are only interested in things that are potentially
+            # python modules or packages, and therefore start with a
+            # letter or _
+            if not entry[0].isalpha() and entry[0] != '_':
+                continue
             entry_path = os.path.join(directory, entry)
             name, ext = os.path.splitext(entry)
             dotted_name = self.dotted_name + '.' + name
             if self.exclude_filter(name):
                 continue
+            if self.ignore_nonsource:
+                if ext in ['.pyo', '.pyc']:
+                    continue
             # Case one: modules
             if (os.path.isfile(entry_path) and ext in ['.py', '.pyc']):
                 if name == '__init__':
@@ -85,28 +96,36 @@ def getSubModuleInfos(self):
                 if name in seen:
                     continue
                 seen.append(name)
-                module_infos.append(ModuleInfo(entry_path, dotted_name,
-                                               exclude_filter=self.exclude_filter))
+                module_infos.append(
+                    ModuleInfo(entry_path,
+                               dotted_name,
+                               exclude_filter=self.exclude_filter,
+                               ignore_nonsource=self.ignore_nonsource)
+                    )
             # Case two: packages
             elif is_package(entry_path):
                 # We can blindly use __init__.py even if only
                 # __init__.pyc exists because we never actually use
                 # that filename.
                 module_infos.append(ModuleInfo(
-                    os.path.join(entry_path, '__init__.py'), dotted_name,
-                                 exclude_filter=self.exclude_filter))
+                    os.path.join(entry_path, '__init__.py'),
+                    dotted_name,
+                    exclude_filter=self.exclude_filter,
+                    ignore_nonsource=self.ignore_nonsource))
         return module_infos
 
     def getSubModuleInfo(self, name):
         path = os.path.join(os.path.dirname(self.path), name)
         if is_package(path):
             return ModuleInfo(os.path.join(path, '__init__.py'),
                               '%s.%s' % (self.package_dotted_name, name),
-                              exclude_filter=self.exclude_filter)
+                              exclude_filter=self.exclude_filter,
+                              ignore_nonsource=self.ignore_nonsource)
         elif os.path.isfile(path + '.py') or os.path.isfile(path + '.pyc'):
                 return ModuleInfo(path + '.py',
                                   '%s.%s' % (self.package_dotted_name, name),
-                                  exclude_filter=self.exclude_filter)
+                                  exclude_filter=self.exclude_filter,
+                                  ignore_nonsource = self.ignore_nonsource)
         else:
             return None
 
@@ -158,18 +177,21 @@ def getResourcePath(self, name):
     def getAnnotation(self, key, default):
         return default
 
-def module_info_from_dotted_name(dotted_name, exclude_filter=None):
+def module_info_from_dotted_name(dotted_name, exclude_filter=None,
+                                 ignore_nonsource=True):
     if dotted_name == '__builtin__':
         # in case of the use of individually grokking something during a
         # test the dotted_name being passed in could be __builtin__
         # in this case we return a special ModuleInfo that just
         # implements enough interface to work
         return BuiltinModuleInfo()
     module = resolve(dotted_name)
-    return ModuleInfo(module.__file__, dotted_name, exclude_filter)
+    return ModuleInfo(module.__file__, dotted_name, exclude_filter,
+                      ignore_nonsource)
 
-def module_info_from_module(module, exclude_filter=None):
-    return ModuleInfo(module.__file__, module.__name__, exclude_filter)
+def module_info_from_module(module, exclude_filter=None, ignore_nonsource=True):
+    return ModuleInfo(module.__file__, module.__name__, exclude_filter,
+                      ignore_nonsource)
 
 
 # taken from zope.dottedname.resolve

src/martian/scan.txt

@@ -194,8 +194,52 @@ nor 'ftests' we could do::
   >>> module_info
   <ModuleInfo object for 'martian.tests.withtestsmodules'>
   >>> no_tests_filter = lambda x: x in ['tests', 'ftests']
-  >>> submodules = module_info.getSubModuleInfos()
-  >>> print submodules
+  >>> print module_info.getSubModuleInfos()
   [<ModuleInfo object for 'martian.tests.withtestsmodules.subpackage'>]
-  >>> print submodules[0].getSubModuleInfos()
-  []
+
+Non-modules that look like modules
+----------------------------------
+
+Sometimes the environment (an editor or network file system, for
+instance) will create a situation where there is a file or directory
+that looks like a Python module or package but is in fact not really
+one (file name starts with a dot, for instance). Module and package
+names must be valid Python identifiers.
+
+The package ``martian.tests.withbogusmodules`` contains only one real
+module and one real package. The rest are almost right, but do not
+start with an underscore and a letter and are therefore not valid. We
+will see that Martian will ignore these other things::
+
+  >>> module_info = module_info_from_dotted_name(
+  ...     'martian.tests.withbogusmodules')
+  >>> module_info.getSubModuleInfos()
+  [<ModuleInfo object for 'martian.tests.withbogusmodules.nonbogus'>,
+   <ModuleInfo object for 'martian.tests.withbogusmodules.subpackage'>]
+
+Packages which contain .pyc files only
+--------------------------------------
+
+When a .py file in a package is renamed, an "orphaned" .pyc object is
+often left around.  By default, Martian will ignore such .pyc files:
+
+  >>> import martian.tests.withpyconly
+  >>> from martian.tests.withpyconly import foo
+  >>> d = os.path.abspath(os.path.dirname(martian.tests.withpyconly.__file__))
+  >>> os.rename(os.path.join(d, 'foo.py'), os.path.join(d, 'foo.py_aside'))
+  >>> module_info = module_info_from_dotted_name(
+  ...     'martian.tests.withpyconly')
+  >>> module_info.getSubModuleInfos()
+  [<ModuleInfo object for 'martian.tests.withpyconly.subpackage'>]
+
+However, if ``ignore_nonsource=False`` is passed to
+``module_info_from_dotted_name`` (or friends, such as
+``module_info_from_module``), we will pick up these modules::
+
+  >>> module_info = module_info_from_dotted_name(
+  ...     'martian.tests.withpyconly', ignore_nonsource=False)
+  >>> module_info.getSubModuleInfos()
+  [<ModuleInfo object for 'martian.tests.withpyconly.foo'>,
+   <ModuleInfo object for 'martian.tests.withpyconly.subpackage'>]
+  >>> # rename back to normal name
+  >>> os.rename(os.path.join(d, 'foo.py_aside'), os.path.join(d, 'foo.py'))

src/martian/tests/withbogusmodules/.bogus.py

@@ -0,0 +1,2 @@
+# starts with a dot so not really a python module
+

src/martian/tests/withbogusmodules/1alsobogus.py

@@ -0,0 +1,2 @@
+# also bogus
+

src/martian/tests/withbogusmodules/nonbogus.py

@@ -0,0 +1 @@
+# really a python module.

src/martian/tests/withpyconly/foo.py

@@ -0,0 +1 @@
+# hello