Merge branch 'master' into fix_config_tests

.travis.yml

@@ -6,7 +6,7 @@ matrix:
   #- python: '2.7'
   #  env: PYRO="Pyro" JYTHON="org.python:jython-installer:2.7.0"
   - python: '3.4'
-    env: PYRO="Pyro4" YAML="pyyaml"
+    env: PYRO="Pyro4" YAML="pyyaml<=5.2"
   - python: '3.5'
     env: PYRO="Pyro4" YAML="pyyaml"
   - python: '3.6'
@@ -23,7 +23,7 @@ matrix:
     env: PYRO="Pyro4" YAML="pyyaml"
 install:
 - if [ -n "${JYTHON}" ]; then source install_jython.sh; fi
-- if [ -n "${YAML}" ]; then pip install pyyaml; fi
+- if [ -n "${YAML}" ]; then pip install "${YAML}"; fi
 - pip install xlrd
 - pip install $PYRO
 - pip install coverage

CHANGELOG.txt

@@ -7,6 +7,12 @@
 -------------------------------------------------------------------------------
 
 
+-------------------------------------------------------------------------------
+ Version 5.7.3   13 Dec 2019
+-------------------------------------------------------------------------------
+
+- Improvements to import_file() and run_file() (#70)
+
 -------------------------------------------------------------------------------
  Version 5.7.2    7 Nov 2019
 -------------------------------------------------------------------------------

RELEASE.txt

@@ -1,4 +1,4 @@
-We are pleased to announce the release of PyUtilib 5.7.2. The
+We are pleased to announce the release of PyUtilib 5.7.3. The
 PyUtilib project supports the development of an ensemble of Python
 packages that include a wide variety of utilities, including a
 well-developed component architecture.

pyutilib/dev/runtests.py

@@ -131,6 +131,14 @@ def run(package, basedir, argv, use_exec=use_exec, env=None):
     else:
         cmd = ['nosetests']
 
+    if (platform == 'win' and sys.version_info[0:2] >= (3, 8)):
+        #######################################################
+        # This option is required due to a (likely) bug within nosetests.
+        # Nose is no longer maintained, but this workaround is based on a public forum suggestion:
+        #   https://stackoverflow.com/questions/58556183/nose-unittest-discovery-broken-on-python-3-8
+        #######################################################
+        cmd.append('--traverse-namespace')
+
     if binDir not in env['PATH']:
         env['PATH'] = os.pathsep.join([binDir, env.get('PATH','')])
 

pyutilib/misc/config.py

@@ -26,13 +26,17 @@ def dump(x, **args):
             return str(x).lower()
         return str(x)
 
-
 try:
     import argparse
     argparse_is_available = True
 except ImportError:
     argparse_is_available = False
 
+try:
+    import builtins as _builtins
+except ImportError:
+    import __builtin__ as _builtins
+
 __all__ = ('ConfigBlock', 'ConfigList', 'ConfigValue')
 
 logger = logging.getLogger('pyutilib.misc.config')
@@ -66,6 +70,35 @@ def _strip_indentation(doc):
     return '\n'.join(lines)
 
 
+def _value2string(prefix, value, obj):
+    _str = prefix
+    if value is not None:
+        try:
+            _data = value._data if value is obj else value
+            if getattr(_builtins, _data.__class__.__name__, None
+                   ) is not None:
+                _str += dump(_data, default_flow_style=True).rstrip()
+                if _str.endswith("..."):
+                    _str = _str[:-3].rstrip()
+            else:
+                _str += str(_data)
+        except:
+            _str += str(type(_data))
+    return _str.rstrip()
+
+def _value2yaml(prefix, value, obj):
+    _str = prefix
+    if value is not None:
+        try:
+            _data = value._data if value is obj else value
+            _str += dump(_data, default_flow_style=True).rstrip()
+            if _str.endswith("..."):
+                _str = _str[:-3].rstrip()
+        except:
+            _str += str(type(_data))
+    return _str.rstrip()
+
+
 class _UnpickleableDomain(object):
     def __init__(self, obj):
         self._type = type(obj).__name__
@@ -390,14 +423,14 @@ def _process_argparse_def(_args, _kwds):
             _parser.add_argument(*_args, default=argparse.SUPPRESS, **_kwds)
 
         assert (argparse_is_available)
-        for level, value, obj in self._data_collector(None, ""):
+        for level, prefix, value, obj in self._data_collector(None, ""):
             if obj._argparse is None:
                 continue
             for _args, _kwds in obj._argparse:
                 _process_argparse_def(_args, _kwds)
 
     def import_argparse(self, parsed_args):
-        for level, value, obj in self._data_collector(None, ""):
+        for level, prefix, value, obj in self._data_collector(None, ""):
             if obj._argparse is None:
                 continue
             for _args, _kwds in obj._argparse:
@@ -421,11 +454,13 @@ def display(self, content_filter=None, indent_spacing=2, ostream=None):
         if ostream is None:
             ostream=stdout
 
-        for level, value, obj in self._data_collector(0, ""):
+        for level, prefix, value, obj in self._data_collector(0, ""):
             if content_filter == 'userdata' and not obj._userSet:
                 continue
 
-            _blocks[level:] = [' ' * indent_spacing * level + value + "\n",]
+            _str = _value2string(prefix, value, obj)
+            _blocks[level:] = [
+                ' ' * indent_spacing * level + _str + "\n",]
 
             for i, v in enumerate(_blocks):
                 if v is not None:
@@ -437,11 +472,12 @@ def generate_yaml_template(self, indent_spacing=2, width=78, visibility=0):
         comment = "  # "
         data = list(self._data_collector(0, "", visibility))
         level_info = {}
-        for lvl, val, obj in data:
+        for lvl, pre, val, obj in data:
+            _str = _value2yaml(pre, val, obj)
             if lvl not in level_info:
                 level_info[lvl] = {'data': [], 'off': 0, 'line': 0, 'over': 0}
             level_info[lvl]['data'].append(
-                (val.find(':') + 2, len(val), len(obj._description or "")))
+                (_str.find(':') + 2, len(_str), len(obj._description or "")))
         for lvl in sorted(level_info):
             indent = lvl * indent_spacing
             _ok = width - indent - len(comment) - minDocWidth
@@ -479,19 +515,20 @@ def generate_yaml_template(self, indent_spacing=2, width=78, visibility=0):
         os = six.StringIO()
         if self._description:
             os.write(comment.lstrip() + self._description + "\n")
-        for lvl, val, obj in data:
+        for lvl, pre, val, obj in data:
+            _str = _value2yaml(pre, val, obj)
             if not obj._description:
-                os.write(' ' * indent_spacing * lvl + val + "\n")
+                os.write(' ' * indent_spacing * lvl + _str + "\n")
                 continue
             if lvl <= maxLvl:
                 field = pad - len(comment)
             else:
                 field = level_info[lvl]['off'] - len(comment)
             os.write(' ' * indent_spacing * lvl)
-            if width - len(val) - minDocWidth >= 0:
-                os.write('%%-%ds' % (field - indent_spacing * lvl) % val)
+            if width - len(_str) - minDocWidth >= 0:
+                os.write('%%-%ds' % (field - indent_spacing * lvl) % _str)
             else:
-                os.write(val + '\n' + ' ' * field)
+                os.write(_str + '\n' + ' ' * field)
             os.write(comment)
             txtArea = max(width - field - len(comment), minDocWidth)
             os.write(("\n" + ' ' * field + comment).join(
@@ -515,7 +552,7 @@ def generate_documentation\
         level = []
         lastObj = self
         indent = ''
-        for lvl, val, obj in self._data_collector(1, '', visibility, True):
+        for lvl, pre, val, obj in self._data_collector(1, '', visibility, True):
             #print len(level), lvl, val, obj
             if len(level) < lvl:
                 while len(level) < lvl - 1:
@@ -575,14 +612,14 @@ def generate_documentation\
     def user_values(self):
         if self._userSet:
             yield self
-        for level, value, obj in self._data_collector(0, ""):
+        for level, prefix, value, obj in self._data_collector(0, ""):
             if obj._userSet:
                 yield obj
 
     def unused_user_values(self):
         if self._userSet and not self._userAccessed:
             yield self
-        for level, value, obj in self._data_collector(0, ""):
+        for level, prefix, value, obj in self._data_collector(0, ""):
             if obj._userSet and not obj._userAccessed:
                 yield obj
 
@@ -605,13 +642,7 @@ def set_value(self, value):
     def _data_collector(self, level, prefix, visibility=None, docMode=False):
         if visibility is not None and visibility < self._visibility:
             return
-        try:
-            _str = dump(self._data, default_flow_style=True).rstrip()
-        except:
-            _str = str(type(self._data))
-        if _str.endswith("..."):
-            _str = _str[:-3].rstrip()
-        yield (level, prefix + _str, self)
+        yield (level, prefix, self, self)
 
 
 class ConfigList(ConfigBase):
@@ -732,7 +763,7 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False):
             # domain, potentially duplicating that documentation is
             # somewhat redundant, and worse, if the list is empty, then
             # no documentation is generated at all!)
-            yield (level, prefix.rstrip(), self)
+            yield (level, prefix, None, self)
             subDomain = self._domain._data_collector(level + 1, '- ',
                                                      visibility, docMode)
             # Pop off the (empty) block entry
@@ -742,9 +773,9 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False):
             return
         if prefix:
             if not self._data:
-                yield (level, prefix.rstrip() + ' []', self)
+                yield (level, prefix, [], self)
             else:
-                yield (level, prefix.rstrip(), self)
+                yield (level, prefix, None, self)
                 if level is not None:
                     level += 1
         for value in self._data:
@@ -1005,7 +1036,7 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False):
         if visibility is not None and visibility < self._visibility:
             return
         if prefix:
-            yield (level, prefix.rstrip(), self)
+            yield (level, prefix, None, self)
             if level is not None:
                 level += 1
         for key in self._decl_order:

pyutilib/misc/import_file.py

@@ -92,11 +92,11 @@ def import_file(filename, context=None, name=None, clear_cache=False):
         else:
             if clear_cache and modulename in sys.modules:
                 del sys.modules[modulename]
-            if dirname is not None:
-                sys.path.insert(0, dirname)
-            else:
-                sys.path.insert(0, implied_dirname)
             try:
+                if dirname is not None:
+                    sys.path.insert(0, dirname)
+                else:
+                    sys.path.insert(0, implied_dirname)
                 module = __import__(modulename)
             except ImportError:
                 pass
@@ -129,8 +129,8 @@ def import_file(filename, context=None, name=None, clear_cache=False):
                                                             [dirname])
                 fp.close()
             else:
-                sys.path.insert(0, implied_dirname)
                 try:
+                    sys.path.insert(0, implied_dirname)
                     # find_module will return the .py file
                     # (never .pyc)
                     fp, pathname, description = imp.find_module(modulename)
@@ -175,7 +175,7 @@ def run_file(filename, logfile=None, execdir=None):
     #
     # Open logfile
     #
-    if not logfile is None:
+    if logfile is not None:
         sys.stderr.flush()
         sys.stdout.flush()
         save_stdout = sys.stdout
@@ -187,44 +187,44 @@ def run_file(filename, logfile=None, execdir=None):
     # Add the file directory to the system path
     #
     currdir_ = ''
-    if '/' in filename:
-        currdir_ = "/".join((filename).split("/")[:-1])
-        tmp_import = (filename).split("/")[-1]
-        sys.path.append(currdir_)
-    elif '\\' in filename:
-        currdir_ = "\\".join((filename).split("\\")[:-1])
-        tmp_import = (filename).split("\\")[-1]
-        sys.path.append(currdir_)
+    norm_file = os.path.normpath(filename)
+    assert norm_file[-1] not in '\\/'
+    split_path = []
+    while norm_file:
+        norm_file, tail = os.path.split(norm_file)
+        split_path.append(tail)
+        # Absolute paths can get stuck returning ('/', '')
+        if not tail:
+            split_path.append(norm_file)
+            norm_file = ''
+    split_path.reverse()
+    if len(split_path) > 1:
+        currdir_ = os.path.join(*tuple(split_path[:-1]))
     else:
-        tmp_import = filename
+        currdir_ = '.'
+    tmp_import = split_path[-1]
+
     name = ".".join((tmp_import).split(".")[:-1])
     #
     # Run the module
     #
     try:
-        if not execdir is None:
+        tmp_path = list(sys.path)
+        if execdir is not None:
             tmp = os.getcwd()
             os.chdir(execdir)
-            tmp_path = sys.path
             sys.path = [execdir] + sys.path
+        # [JDS 191130] I am not sure why we put the target file's
+        # directory at the end of sys.path, but I am preserving that
+        # decision.
+        sys.path.append(currdir_)
         runpy.run_module(name, None, "__main__")
-        if not execdir is None:
+    finally:
+        # Mandatory cleanup
+        sys.path = tmp_path
+        if execdir is not None:
             os.chdir(tmp)
-            sys.path = tmp_path
-    except Exception:  #pragma:nocover
-        if not logfile is None:
+        if logfile is not None:
             OUTPUT.close()
             sys.stdout = save_stdout
             sys.stderr = save_stderr
-        raise
-    if currdir_ in sys.path:
-        sys.path.remove(currdir_)
-    if execdir in sys.path:
-        sys.path.remove(execdir)
-    #
-    # Close logfile
-    #
-    if not logfile is None:
-        OUTPUT.close()
-        sys.stdout = save_stdout
-        sys.stderr = save_stderr

pyutilib/misc/tests/import_exception.py

@@ -0,0 +1,3 @@
+import sys
+
+raise RuntimeError("raised during import")

pyutilib/misc/tests/import_main_exception.py

@@ -0,0 +1,5 @@
+import sys
+
+if __name__ == "__main__":
+    print("import_main_exception - main")
+    raise RuntimeError("raised from __main__")

pyutilib/misc/tests/import_main_exception.txt

@@ -0,0 +1 @@
+import_main_exception - main