DM-11514: Use pytest as simple test runner

.gitignore

@@ -1,3 +1,4 @@
+.cache
 .sconsign.dblite
 config.log
 .sconf_temp

bin/makeLinkFarm

@@ -1,8 +1,12 @@
 #!/usr/bin/env python
 
-import argparse, os, re, sys
+from __future__ import print_function
+import argparse
+import os
+import sys
 import eups
 
+
 def linkFiles(dstDir, srcDir, verbose=0, force=False):
     """Link all the regular files in srcDir into dstDir; mkdir all the directories in srcDir and
     descend recursively"""
@@ -18,17 +22,18 @@ def linkFiles(dstDir, srcDir, verbose=0, force=False):
             linkFiles(fullDst, fullSrc, verbose, force)
         else:
             if verbose > 1:
-                print "%s -> %s" % (fullSrc, fullDst)
+                print("%s -> %s" % (fullSrc, fullDst))
 
             if os.path.exists(fullDst):
                 if force:
                     os.unlink(fullDst)
                 else:
-                    print >> sys.stderr, "%s already exists; use --force to overwrite" % fullDst
+                    print("%s already exists; use --force to overwrite" % fullDst, file=sys.stderr)
                     sys.exit(1)
 
             os.symlink(fullSrc, fullDst)
 
+
 def main():
     parser = argparse.ArgumentParser(description='''Build a link farm to replace long -I/-L argument lists.
 All the bin, include files, and libraries for all the dependencies of the specified product are linked into
@@ -44,13 +49,13 @@ creates a link farm for your current selection of products.
 
     parser.add_argument('--force', '-f', action='store_true', help="Force creation of farm even if it exists")
     parser.add_argument('--verbose', '-v', action='count', help="Be chatty")
-    
+
     args = parser.parse_args()
 
     linkFarmDir = os.path.abspath(args.linkFarmDir)
 
     if not args.force and os.path.exists(linkFarmDir):
-        print >> sys.stderr, "Path %s already exists; use --force to overwrite" % args.linkFarmDir
+        print("Path %s already exists; use --force to overwrite" % args.linkFarmDir, file=sys.stderr)
         sys.exit(1)
 
     #
@@ -64,23 +69,23 @@ creates a link farm for your current selection of products.
     products = {}
     for productName, versionName, optional, recursionDepth in \
             eups.getDependencies(args.productName, None, setup=True):
-        if products.has_key(productName):
+        if productName in products:
             continue
 
         products[productName] = 1
 
         pd = eups.productDir(productName)
         if not pd:                      # no product
             continue
-        
+
         binDir = os.path.join(pd, "bin")
         incDir = os.path.join(pd, "include")
         libDir = os.path.join(pd, "lib")
 
         if args.verbose:
             if args.verbose > 1 or \
                     os.path.exists(binDir) or os.path.exists(incDir) or os.path.exists(libDir):
-                print "%-15s %s" % (productName, pd)
+                print("%-15s %s" % (productName, pd))
 
         if os.path.exists(binDir):
             fullDstDir = os.path.join(linkFarmDir, "bin")
@@ -92,22 +97,22 @@ creates a link farm for your current selection of products.
 
         if os.path.exists(libDir):
             for src in os.listdir(libDir):
-                if not re.search(r"^lib", src):
+                if not src.startswith("lib"):
                     continue
 
                 fullSrc = os.path.join(libDir, src)
                 fullDst = os.path.join(linkFarmDir, "lib", src)
                 if args.verbose > 1:
-                    print src, fullSrc, fullDst
+                    print(src, fullSrc, fullDst)
                 if os.path.exists(fullDst):
                     if args.force:
                         os.unlink(fullDst)
                     else:
-                        print >> sys.stderr, "%s already exists; use --force to overwrite" % fullDst
+                        print("%s already exists; use --force to overwrite" % fullDst, file=sys.stderr)
                         sys.exit(1)
 
                 os.symlink(fullSrc, fullDst)
 
+
 if __name__ == "__main__":
     main()
-

bin/sconsOpts

@@ -2,12 +2,15 @@
 #
 # Print the options last passed to scons
 #
-import os, sys
+from __future__ import print_function
+import os
+import sys
 import ConfigParser
 import argparse
 
 parser = argparse.ArgumentParser(description="Tell us how scons was invoked")
-parser.add_argument('configFile', type=str, nargs="?", default=None, help='The file with the information we need')
+parser.add_argument('configFile', type=str, nargs="?", default=None,
+                    help='The file with the information we need')
 parser.add_argument('--cc', type=str, default="gcc",
                     help="Use this compiler if build.cfg is unavailable")
 parser.add_argument('--opt', type=int, default=0,
@@ -17,28 +20,27 @@ parser.add_argument('--quiet', '-q', action="store_true", help="Don't generate a
 args = parser.parse_args()
 dirName = "."
 if args.configFile and os.path.isdir(args.configFile):
-   dirName, args.configFile = args.configFile, None
+    dirName, args.configFile = args.configFile, None
 
 if not args.configFile:
-   args.configFile = os.path.join(dirName, ".sconf_temp", "build.cfg")
+    args.configFile = os.path.join(dirName, ".sconf_temp", "build.cfg")
 
 cc = args.cc
 opt = args.opt
 
 if os.path.exists(args.configFile):
-   config = ConfigParser.ConfigParser()
-   try:
-      config.read(args.configFile)
+    config = ConfigParser.ConfigParser()
+    try:
+        config.read(args.configFile)
 
-      cc = config.get("Build", 'cc')
-      opt = config.get("Build", 'opt', 'opt')
-   except Exception, e:
-      if not args.quiet:
-         print >> sys.stderr, "File %s: error %s" % (args.configFile, e)
+        cc = config.get("Build", 'cc')
+        opt = config.get("Build", 'opt', 'opt')
+    except Exception as e:
+        if not args.quiet:
+            print("File %s: error %s" % (args.configFile, e), file=sys.stderr)
 
 else:
-   if not args.quiet:
-      print >> sys.stderr, "File %s doesn't exist" % args.configFile
+    if not args.quiet:
+        print("File %s doesn't exist" % args.configFile, file=sys.stderr)
 
-print "cc=%s opt=%s" % (cc, opt)
-
+print("cc=%s opt=%s" % (cc, opt))

python/lsst/sconsUtils/builders.py

@@ -117,7 +117,7 @@ def SourcesForSharedLibrary(self, files):
             obj = self.SharedObject(ccFile)
         objs.append(obj)
 
-    objs = sorted(state.env.Flatten(sources), key=str)
+    objs = sorted(state.env.Flatten(objs), key=str)
     return objs
 
 
@@ -155,7 +155,7 @@ def filesToTag(root=None, fileRegex=None, ignoreDirs=None):
         #
         # Remove files generated by swig
         #
-        for swigFile in [f for f in filenames if re.search(r"\.i$", f)]:
+        for swigFile in [f for f in filenames if f.endswith(".i")]:
             name = os.path.splitext(swigFile)[0]
             candidates = [f for f in candidates if not re.search(r"%s(_wrap\.cc?|\.py)$" % name, f)]
 

python/lsst/sconsUtils/dependencies.py

@@ -587,4 +587,4 @@ def getLibs(env, categories="main"):
 
 SConsEnvironment.getLibs = getLibs
 
-## @}
+## @}  # noqa E266

python/lsst/sconsUtils/eupsForScons.py

@@ -2,7 +2,10 @@
 import os
 
 try:
-    from eups import *
+    # Prefer to use native EUPS but if that is not available, fallback
+    # versions are defined. Only a subset of EUPS functions are required
+    # but all are imported to prevent warnings from redefinitions below.
+    from eups import *  # noqa F403 F401
     eupsLoaded = True
 except ImportError:
     eupsLoaded = False

python/lsst/sconsUtils/installation.py

@@ -335,7 +335,7 @@ def InstallEups(env, dest, files=[], presetup=""):
 
             path = eups.Eups.setEupsPath()
             if path:
-                locks = eups.lock.takeLocks("setup", path, eups.lock.LOCK_SH)
+                locks = eups.lock.takeLocks("setup", path, eups.lock.LOCK_SH)  # noqa F841 keep locks active
                 env["ENV"]["EUPS_LOCK_PID"] = os.environ.get("EUPS_LOCK_PID", "-1")
         except ImportError:
             state.log.warn("Unable to import eups; not locking")

python/lsst/sconsUtils/scripts.py

@@ -195,7 +195,16 @@ def finish(defaultTargets=DEFAULT_TARGETS,
                       nfail=`find {0} -name \*.failed | wc -l | sed -e 's/ //g'`; \
                       if [ $$nfail -gt 0 ]; then \
                           echo "Failed test output:" >&2; \
-                          find {0} -name \*.failed -exec cat {{}} \; >&2; \
+                          for f in `find {0} -name \*.failed`; do \
+                              case "$$f" in \
+                              *.xml.failed) \
+                                echo "Global pytest output is in $$f" >&2; \
+                                ;; \
+                              *.failed) \
+                                cat $$f >&2; \
+                                ;; \
+                              esac; \
+                          done; \
                           echo "The following tests failed:" >&2;\
                           find {0} -name \*.failed >&2; \
                           echo "$$nfail tests failed" >&2; exit 1; \
@@ -299,8 +308,10 @@ def rewrite_shebang(target, source, env):
         if src is None:
             src = Glob("#bin.src/*")
         for s in src:
-            if str(s) != "SConscript":
-                result = state.env.Command(target=os.path.join(Dir("#bin").abspath, str(s)),
+            filename = str(s)
+            # Do not try to rewrite files starting with non-letters
+            if filename != "SConscript" and re.match("[A-Za-z]", filename):
+                result = state.env.Command(target=os.path.join(Dir("#bin").abspath, filename),
                                            source=s, action=rewrite_shebang)
                 state.targets["shebang"].extend(result)
 
@@ -424,6 +435,13 @@ def doc(config="doxygen.conf.in", projectName=None, projectNumber=None, **kw):
     #  @param pyList           A sequence of Python tests to run (including .py extensions).
     #                          Defaults to a *.py glob of the tests directory, minus any
     #                          files corresponding to the SWIG modules in swigFileList.
+    #                          An empty list will enable automated test discovery.
+    #  @param pySingles        A sequence of Python tests to run (including .py extensions)
+    #                          as independent single tests. By default this list is empty
+    #                          and all tests are run in a single pytest call.
+    #                          Items specified here will not appear in the default pyList
+    #                          and should not start with "test_" (such that they will not
+    #                          be auto-discoverable by pytest).
     #  @param ccList           A sequence of C++ unit tests to run (including .cc extensions).
     #                          Defaults to a *.cc glob of the tests directory, minus any
     #                          files that end with *_wrap.cc and files present in swigSrc.
@@ -439,10 +457,12 @@ def doc(config="doxygen.conf.in", projectName=None, projectNumber=None, **kw):
     ##
     @staticmethod
     def tests(pyList=None, ccList=None, swigNameList=None, swigSrc=None,
-              ignoreList=None, noBuildList=None,
+              ignoreList=None, noBuildList=None, pySingles=None,
               args=None):
         if noBuildList is None:
             noBuildList = []
+        if pySingles is None:
+            pySingles = []
         if swigNameList is None:
             swigFileList = Glob("*.i")
             swigNameList = [_getFileBase(node) for node in swigFileList]
@@ -459,6 +479,10 @@ def tests(pyList=None, ccList=None, swigNameList=None, swigSrc=None,
             pyList = [node for node in Glob("*.py")
                       if _getFileBase(node) not in swigNameList and
                       os.path.basename(str(node)) not in noBuildList]
+            # if we got no matches, reset to None so we do not enabled
+            # auto test detection in pytest
+            if not pyList:
+                pyList = None
         if ccList is None:
             ccList = [node for node in Glob("*.cc")
                       if (not str(node).endswith("_wrap.cc")) and str(node) not in allSwigSrc and
@@ -467,7 +491,10 @@ def tests(pyList=None, ccList=None, swigNameList=None, swigSrc=None,
             ignoreList = []
 
         def s(l):
+            if l is None:
+                return ['None']
             return [str(i) for i in l]
+
         state.log.info("SWIG modules for tests: %s" % s(swigFileList))
         state.log.info("Python tests: %s" % s(pyList))
         state.log.info("C++ tests: %s" % s(ccList))
@@ -481,9 +508,31 @@ def s(l):
             swigMods.extend(
                 state.env.SwigLoadableModule("_" + name, src, LIBS=state.env.getLibs("main python"))
             )
+
+        # Warn about insisting that a test in pySingles starts with test_ and
+        # therefore might be automatically discovered by pytest. These files
+        # should not be discovered automatically.
+        for node in pySingles:
+            if str(node).startswith("test_"):
+                state.log.warn("Warning: {} should be run independently but"
+                               " can be automatically discovered".format(node))
+
+        # Ensure that python tests listed in pySingles are not included in pyList.
+        if pyList is not None:
+            pyList = [str(node) for node in pyList if str(node) not in pySingles]
+
         ccList = [control.run(str(node)) for node in ccList]
-        pyList = [control.run(str(node)) for node in pyList]
+        pySingles = [control.run(str(node)) for node in pySingles]
+
+        # If we tried to discover .py files and found none, do not then
+        # try to use auto test discovery.
+        if pyList is not None:
+            pyList = [control.runPythonTests(pyList)]
+        else:
+            pyList = []
+        pyList.extend(pySingles)
         for pyTest in pyList:
+            state.env.Depends(pyTest, ccList)
             state.env.Depends(pyTest, swigMods)
             state.env.Depends(pyTest, state.targets["python"])
             state.env.Depends(pyTest, state.targets["shebang"])
@@ -533,4 +582,4 @@ def examples(ccList=None, swigNameList=None, swigSrc=None):
         state.targets["examples"].extend(results)
         return results
 
-## @}
+## @}  # noqa E266