ARELLE 348 - plug ins, add examples, test subdirectories, test packag…

…e relative imports

arelle/PluginManager.py

@@ -142,6 +142,7 @@ def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False):
             # if moduleFilename is a directory containing an __ini__.py file, open that instead
             if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")):
                 moduleFilename = os.path.join(moduleFilename, "__init__.py")
+            moduleDir = os.path.dirname(moduleFilename)
             f = openFileStream(_cntlr, moduleFilename)
             tree = ast.parse(f.read(), filename=moduleFilename)
             moduleImports = []
@@ -198,13 +199,23 @@ def moduleModuleInfo(moduleURL, reload=False, parentImportsSubtree=False):
                         moduleInfo["imports"] =  imports
                         return moduleInfo
                 elif isinstance(item, ast.ImportFrom):
-                    if item.module is None:  # from . import module1, module2, ...
-                        for importee in item.names:
-                            if importee.name not in moduleImports:
-                                moduleImports.append(importee.name)
-                    elif '.' not in item.module: # from .module import method1, method2, ...
-                        if item.module not in moduleImports:
-                            moduleImports.append(item.module)
+                    if item.level == 1: # starts with .
+                        if item.module is None:  # from . import module1, module2, ...
+                            for importee in item.names:
+                                if (os.path.isfile(os.path.join(moduleDir, importee.name + ".py"))
+                                    and importee.name not in moduleImports):
+                                    moduleImports.append(importee.name)
+                        else:
+                            modulePkgs = item.module.split('.')
+                            modulePath = os.path.join(*modulePkgs)
+                            if (os.path.isfile(os.path.join(moduleDir, modulePath) + ".py")
+                                and modulePath not in moduleImports):
+                                    moduleImports.append(modulePath)
+                            for importee in item.names:
+                                _importeePfxName = os.path.join(modulePath, importee.name)
+                                if (os.path.isfile(os.path.join(moduleDir, _importeePfxName) + ".py")
+                                    and _importeePfxName not in moduleImports):
+                                        moduleImports.append(_importeePfxName)
         except EnvironmentError:
             pass
         if f:
@@ -219,7 +230,7 @@ def moduleInfo(pluginInfo):
         elif isinstance(value, types.FunctionType):
             moduleInfo.getdefault('classes',[]).append(name)
 
-def loadModule(moduleInfo):
+def loadModule(moduleInfo, packagePrefix=""):
     name = moduleInfo['name']
     moduleURL = moduleInfo['moduleURL']
     moduleFilename = _cntlr.webCache.getfilename(moduleURL, normalize=True, base=_pluginBase)
@@ -230,13 +241,15 @@ def loadModule(moduleInfo):
             if os.path.isdir(moduleFilename) and os.path.isfile(os.path.join(moduleFilename, "__init__.py")):
                 moduleDir = os.path.dirname(moduleFilename)
                 moduleName = os.path.basename(moduleFilename)
+                packageImportPrefix = moduleName + "."
             else:
                 moduleName = os.path.basename(moduleFilename).partition('.')[0]
                 moduleDir = os.path.dirname(moduleFilename)
+                packageImportPrefix = packagePrefix
             file, path, description = imp.find_module(moduleName, [moduleDir])
             if file or path: # file returned if non-package module, otherwise just path for package
                 try:
-                    module = imp.load_module(moduleName, file, path, description)
+                    module = imp.load_module(packagePrefix + moduleName, file, path, description)
                     pluginInfo = module.__pluginInfo__.copy()
                     elementSubstitutionClasses = None
                     if name == pluginInfo.get('name'):
@@ -272,8 +285,8 @@ def loadModule(moduleInfo):
                             print(_("Exception loading plug-in {name}: processing ModelObjectFactory.ElementSubstitutionClasses").format(
                                     name=name, error=err), file=sys.stderr)
                     for importModuleInfo in moduleInfo.get('imports', EMPTYLIST):
-                        loadModule(importModuleInfo)
-                except (ImportError, AttributeError) as err:
+                        loadModule(importModuleInfo, packageImportPrefix)
+                except (ImportError, AttributeError, SystemError) as err:
                     print(_("Exception loading plug-in {name}: {error}").format(
                             name=name, error=err), file=sys.stderr)
                 finally:

arelle/examples/plugin/importTestChild1.py

@@ -5,16 +5,16 @@
 '''
 
 def foo():
-	print ("imported plug-in child 1")
+	print ("imported unpackaged plug-in child 1")
 
 __pluginInfo__ = {
-    'name': 'Import Test Child 1',
+    'name': 'Unpackaged Listed Import Child 1',
     'version': '0.9',
-    'description': "This is a packages-containing child plugin.",
+    'description': "This is a unpackaged child plugin.",
     'license': 'Apache-2',
     'author': 'Mark V Systems',
     'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
     # classes of mount points (required)
-    'Import.Example.Entry2': foo,
+    'Import.Unpackaged.Entry2': foo,
     # imported plugins
 }

arelle/examples/plugin/importTestChild2.py

@@ -5,17 +5,17 @@
 '''
 
 def foo():
-	print ("imported plug-in child 2")
+	print ("imported unpackaged plug-in child 2")
 
 __pluginInfo__ = {
-    'name': 'Import Test Child 2',
+    'name': 'Unpackaged Listed Import Child 2',
     'version': '0.9',
-    'description': "This is a packages-containing child plugin.",
+    'description': "This is a unpackaged child plugin.",
     'license': 'Apache-2',
     'author': 'Mark V Systems',
     'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
     # classes of mount points (required)
-    'Import.Example.Entry3': foo,
+    'Import.Unpackaged.Entry3': foo,
     # import plugins
     'import': ('importTestGrandchild1.py', 'importTestGrandchild2.py')
 }

arelle/examples/plugin/importTestGrandchild1.py

@@ -5,16 +5,16 @@
 '''
 
 def foo():
-	print ("imported plug-in grandchild 1")
+	print ("imported unpackaged plug-in grandchild 1")
 
 __pluginInfo__ = {
-    'name': 'Import Test Grandchild 1.1',
+    'name': 'Unpackaged Listed Import Grandchild 1.1',
     'version': '0.9',
-    'description': "This is a packages-containing child plugin.",
+    'description': "This is a packages-containing unpackaged child plugin.",
     'license': 'Apache-2',
     'author': 'Mark V Systems',
     'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
     # classes of mount points (required)
-    'Import.Example.Entry4': foo,
+    'Import.Unpackaged.Entry4': foo,
     # imported plugins
 }

arelle/examples/plugin/importTestGrandchild2.py

@@ -5,16 +5,16 @@
 '''
 
 def foo():
-	print ("imported plug-in grandchild 2")
+	print ("imported unpackaged plug-in grandchild 2")
 
 __pluginInfo__ = {
-    'name': 'Import Test Grandchild 1.2',
+    'name': 'Unpackaged Listed Import Grandchild 1.2',
     'version': '0.9',
     'description': "This is a packages-containing child plugin.",
     'license': 'Apache-2',
     'author': 'Mark V Systems',
     'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
     # classes of mount points (required)
-    'Import.Example.Entry5': foo,
+    'Import.Unpackaged.Entry5': foo,
     # imported plugins
 }

arelle/examples/plugin/importTestImported1.py

@@ -0,0 +1,22 @@
+'''
+pluginPackages test case
+
+(c) Copyright 2012 Mark V Systems Limited, All rights reserved.
+'''
+# this module would raise system error due to PEP 366 after python 3.4.3
+from . import importTestImported11
+
+def foo():
+	print ("imported unpackaged plug-in relative imported 1")
+
+__pluginInfo__ = {
+    'name': 'Unpackaged Relative Import 1',
+    'version': '0.9',
+    'description': "This is a unpackaged relative imported plugin.",
+    'license': 'Apache-2',
+    'author': 'Mark V Systems',
+    'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
+    # classes of mount points (required)
+    'Import.Unpackaged.Entry6': foo,
+    # imported plugins
+}

arelle/examples/plugin/importTestImported11.py

@@ -0,0 +1,21 @@
+'''
+pluginPackages test case
+
+(c) Copyright 2012 Mark V Systems Limited, All rights reserved.
+'''
+# this module would raise system error due to PEP 366 after python 3.4.3
+
+def foo():
+	print ("imported unpackaged plug-in imported relative 1.1")
+
+__pluginInfo__ = {
+    'name': 'Unpackaged Relative Import 1.1',
+    'version': '0.9',
+    'description': "This is a packages-containing unpackaged imported plugin.",
+    'license': 'Apache-2',
+    'author': 'Mark V Systems',
+    'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
+    # classes of mount points (required)
+    'Import.Unpackaged.Entry7': foo,
+    # imported plugins
+}

arelle/examples/plugin/importTestParent.py

@@ -4,39 +4,44 @@
 (c) Copyright 2012 Mark V Systems Limited, All rights reserved.
 '''
 from arelle.PluginManager import pluginClassMethods
+# . relative import only works inside a package now, see https://www.python.org/dev/peps/pep-0366/
+# following two imports raise system error due to PEP 366 after python 3.4.3
+# from . import importTestImported1
+# from .importTestImported1 import foo
 
 def parentMenuEntender(cntlr, menu):
-    menu.add_command(label="Parent exercise descendants", underline=0, command=lambda: parentMenuCommand(cntlr) )
+    menu.add_command(label="Unpackaged Parent exercise descendants", underline=0, command=lambda: parentMenuCommand(cntlr) )
 
 def parentMenuCommand(cntl):
-    for i in range(1,6):
-        for pluginMethod in pluginClassMethods("Import.Example.Entry{}".format(i)):
+    for i in range(1,100):
+        for pluginMethod in pluginClassMethods("Import.Unpackaged.Entry{}".format(i)):
             pluginMethod()
 
 def parentCommandLineOptionExtender(parser):
-    parser.add_option("--parentImportExample", 
+    parser.add_option("--unpackageParentImportExample", 
                       action="store_true", 
-                      dest="parentImportExample", 
-                      help=_('Test that imported plug-ins were actually loaded and activated"'))
+                      dest="unpackageParentImportExample", 
+                      help=_('Test that unpackaged imported plug-ins were actually loaded and activated"'))
 
 def parentCommandLineUtilityRun(cntlr, options, **kwargs):
-    parentMenuCommand(cntlr)
+	if options.unpackageParentImportExample:
+	    parentMenuCommand(cntlr)
 
 def foo():
-	print ("parent of imported plug-ins")
+	print ("parent of imported unpackaged plug-ins")
 
 __pluginInfo__ = {
-    'name': 'Import Test Parent',
+    'name': 'Import Test Unpackaged Parent',
     'version': '0.9',
-    'description': "This is a imports-containing parent plugin.",
+    'description': "This is a imports-containing unpackaged parent plugin.",
     'license': 'Apache-2',
     'author': 'Mark V Systems',
     'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
     # classes of mount points (required)
     'CntlrWinMain.Menu.Tools': parentMenuEntender,
     'CntlrCmdLine.Options': parentCommandLineOptionExtender,
     'CntlrCmdLine.Utility.Run': parentCommandLineUtilityRun,
-    'Import.Example.Entry1': foo,
+    'Import.Unpackaged.Entry1': foo,
     # imported plugins
-    'import': ('importTestChild1.py', 'importTestChild2.py')
+    'import': ('importTestChild1.py', 'importTestChild2.py', "module_import_subtree")
 }

arelle/examples/plugin/packagedImportTest/__init__.py

@@ -0,0 +1,46 @@
+'''
+pluginPackages test case
+
+(c) Copyright 2012 Mark V Systems Limited, All rights reserved.
+'''
+from os import path
+from arelle.PluginManager import pluginClassMethods
+from . import importTestImported1
+from .importTestImported1 import foo
+
+def parentMenuEntender(cntlr, menu):
+    menu.add_command(label="Packaged Parent exercise descendants", underline=0, command=lambda: parentMenuCommand(cntlr) )
+
+def parentMenuCommand(cntl):
+    for i in range(1,100):
+        for pluginMethod in pluginClassMethods("Import.Packaged.Entry{}".format(i)):
+            pluginMethod()
+
+def parentCommandLineOptionExtender(parser):
+    parser.add_option("--packagedParentImportExample", 
+                      action="store_true", 
+                      dest="packagedParentImportExample", 
+                      help=_('Test that imported plug-ins were actually loaded and activated"'))
+
+def parentCommandLineUtilityRun(cntlr, options, **kwargs):
+	if options.packagedParentImportExample:
+	    parentMenuCommand(cntlr)
+
+def foo():
+	print ("parent (__init__) of imported packaged plug-ins")
+
+__pluginInfo__ = {
+    'name': 'Import Test Package Parent',
+    'version': '0.9',
+    'description': "This is a imports-containing packaged (__init__) plugin.",
+    'license': 'Apache-2',
+    'author': 'Mark V Systems',
+    'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
+    # classes of mount points (required)
+    'CntlrWinMain.Menu.Tools': parentMenuEntender,
+    'CntlrCmdLine.Options': parentCommandLineOptionExtender,
+    'CntlrCmdLine.Utility.Run': parentCommandLineUtilityRun,
+    'Import.Packaged.Entry1': foo,
+    # imported plugins
+    'import': ('importTestChild1.py', 'importTestChild2.py', "module_import_subtree")
+}

arelle/examples/plugin/packagedImportTest/importTestChild1.py

@@ -0,0 +1,20 @@
+'''
+pluginPackages test case
+
+(c) Copyright 2012 Mark V Systems Limited, All rights reserved.
+'''
+
+def foo():
+	print ("imported packaged plug-in child 1")
+
+__pluginInfo__ = {
+    'name': 'Package Listed Import Child 1',
+    'version': '0.9',
+    'description': "This is a packaged child plugin.",
+    'license': 'Apache-2',
+    'author': 'Mark V Systems',
+    'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
+    # classes of mount points (required)
+    'Import.Packaged.Entry2': foo,
+    # imported plugins
+}

arelle/examples/plugin/packagedImportTest/importTestChild2.py

@@ -0,0 +1,21 @@
+'''
+pluginPackages test case
+
+(c) Copyright 2012 Mark V Systems Limited, All rights reserved.
+'''
+
+def foo():
+	print ("imported packaged plug-in child 2")
+
+__pluginInfo__ = {
+    'name': 'Package Listed Import Child 2',
+    'version': '0.9',
+    'description': "This is a packaged child plugin.",
+    'license': 'Apache-2',
+    'author': 'Mark V Systems',
+    'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
+    # classes of mount points (required)
+    'Import.Packaged.Entry3': foo,
+    # import plugins
+    'import': ('importTestGrandchild1.py', 'importTestGrandchild2.py')
+}

arelle/examples/plugin/packagedImportTest/importTestGrandchild1.py

@@ -0,0 +1,20 @@
+'''
+pluginPackages test case
+
+(c) Copyright 2012 Mark V Systems Limited, All rights reserved.
+'''
+
+def foo():
+	print ("imported packaged plug-in grandchild 1")
+
+__pluginInfo__ = {
+    'name': 'Package Listed Import Grandchild 1.1',
+    'version': '0.9',
+    'description': "This is a packaged grandchild plugin.",
+    'license': 'Apache-2',
+    'author': 'Mark V Systems',
+    'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
+    # classes of mount points (required)
+    'Import.Packaged.Entry4': foo,
+    # imported plugins
+}

arelle/examples/plugin/packagedImportTest/importTestGrandchild2.py

@@ -0,0 +1,20 @@
+'''
+pluginPackages test case
+
+(c) Copyright 2012 Mark V Systems Limited, All rights reserved.
+'''
+
+def foo():
+	print ("imported packaged plug-in grandchild 2")
+
+__pluginInfo__ = {
+    'name': 'Package Listed Import Grandchild 1.2',
+    'version': '0.9',
+    'description': "This is a packaged grandchild plugin.",
+    'license': 'Apache-2',
+    'author': 'Mark V Systems',
+    'copyright': '(c) Copyright 2015 Mark V Systems Limited, All rights reserved.',
+    # classes of mount points (required)
+    'Import.Packaged.Entry5': foo,
+    # imported plugins
+}