Merge pull request #1181 from ericmehl/IECore-Windows-Tests

IECorePython Windows tests

Author: johnhaddon

.github/workflows/main.yml

@@ -82,6 +82,7 @@ jobs:
             buildType: RELEASE
             options: .github/workflows/main/options.windows
             dependenciesURL: "https://github.com/hypothetical-inc/gafferDependencies/releases/download/3.0.0a1/gafferDependencies-3.0.0a1-Python3-windows.zip"
+            tests: testCorePython
             publish: false
 
     runs-on: ${{ matrix.os }}
@@ -157,10 +158,6 @@ jobs:
         scons ${{ matrix.tests }} BUILD_TYPE=${{ matrix.buildType }} OPTIONS=${{ matrix.options }} CXXSTD=c++14 BUILD_CACHEDIR=sconsCache
       env:
         IECORE_RTLD_GLOBAL: 0
-      # We have some work to do before we can run the tests
-      # successfully on Windows. Until then, we satisfy ourselves
-      # with checking that it at least _builds_.
-      if: runner.os != 'Windows'
 
     - name: Build Package
       run: |

.github/workflows/main/options.windows

@@ -24,6 +24,8 @@ CXXFLAGS = ["/permissive-", "/D_USE_MATH_DEFINES", "/Zc:externC-"]
 
 LIBPATH = deps + "\\lib"
 
+TEST_LIBPATH = deps + "\\lib"
+
 PYTHON = deps + "\\bin\\python"
 PYTHON_INCLUDE_PATH = deps + "\\include"
 PYTHON_LINK_FLAGS = ""
@@ -59,5 +61,7 @@ APPLESEED_ROOT = deps + "\\appleseed"
 APPLESEED_INCLUDE_PATH = deps + "\\appleseed\\include"
 APPLESEED_LIB_PATH = deps + "\\appleseed\\lib"
 
-ENV_VARS_TO_IMPORT = "PATH"
+# PATH is needed to locate the MSVC compiler. PROCESSOR_ARCHITECTURE and USERNAME
+# are needed by the HeaderGenerator and accompanying tests.
+ENV_VARS_TO_IMPORT = "PATH PROCESSOR_ARCHITECTURE USERNAME"
 

Changes

@@ -1,3 +1,5 @@
+
+
 10.3.x.x (relative to 10.2.x.x)
 ========
 
@@ -6,6 +8,11 @@ Improvements
 
 - IECoreArnold : Prepare for multiple universes (#1171)
 
+Breaking Changes
+----------------
+- HeaderGenerator : `machineName`, `systemRelease`, `systemVersion`, may differ from previous versions on Windows. Linux and MacOS are unaffected.
+- HeaderGenerator : `nodeName` is no longer converted to all capital letters and may differ from previous versions on Windows. Linux and MacOS are unaffected.
+
 10.2.x.x (relative to 10.2.2.0)
 ========
 

SConstruct

@@ -274,19 +274,19 @@ o.Add(
 
 o.Add(
 	"CPPPATH",
-	"A colon separated list of paths to search for headers on.",
+	"A colon (semi-colon on Windows) separated list of paths to search for headers on.",
 	"",
 )
 
 o.Add(
 	"LIBPATH",
-	"A colon separated list of paths to search for libraries on.",
+	"A colon (semi-colon on Windows) separated list of paths to search for libraries on.",
 	"",
 )
 
 o.Add(
 	"PYTHONPATH",
-	"A colon separated list of paths to search for python modules on.",
+	"A colon (semi-colon on Windows) separated list of paths to search for python modules on.",
 	"",
 )
 
@@ -991,8 +991,8 @@ o.Add(
 
 o.Add(
 	"TEST_LIBPATH",
-	"Additional colon separated paths to be prepended to the library path"
-	"used when running tests.",
+	"Additional colon separated (semi-colon on Windows) paths to be prepended to the "
+	"library path used when running tests.",
 	""
 )
 
@@ -1001,17 +1001,14 @@ if Environment()["PLATFORM"]=="darwin" :
 elif Environment()["PLATFORM"] != "win32":
 	libraryPathEnvVar = "LD_LIBRARY_PATH"
 else:
-	# Windows. We don't have a library path, but we
-	# can't use an empty string for an environment
-	# variable name.
-	libraryPathEnvVar = "UNUSED_LIBRARY_PATH"
+	libraryPathEnvVar = "PATH"
 
 o.Add(
 	"TEST_LIBRARY_PATH_ENV_VAR",
 	"This is a curious one, probably only ever necessary at image engine. It "
 	"specifies the name of an environment variable used to specify the library "
 	"search paths correctly when running the tests. Defaults to LD_LIBRARY_PATH on "
-	"Linux and DYLD_LIBRARY_PATH on OSX.",
+	"Linux, DYLD_LIBRARY_PATH on OSX and PATH on Windows.",
 	libraryPathEnvVar
 )
 
@@ -1036,7 +1033,7 @@ env = Environment(
 )
 
 if isinstance( env["LIBPATH"], str ) :
-	env["LIBPATH"] = env["LIBPATH"].split( ":" )
+	env["LIBPATH"] = env["LIBPATH"].split( os.pathsep )
 
 for e in env["ENV_VARS_TO_IMPORT"].split() :
 	if e in os.environ :
@@ -1482,25 +1479,32 @@ if pythonModuleEnv["PLATFORM"]=="darwin" :
 
 testEnv = env.Clone()
 
-testEnvLibPath = ":".join( testEnv["LIBPATH"] )
+testEnvLibPath = os.pathsep.join( testEnv["LIBPATH"] )
 if testEnv["TEST_LIBPATH"] != "" :
-	testEnvLibPath = testEnv["TEST_LIBPATH"] + ":" + testEnvLibPath
+	testEnvLibPath = testEnv["TEST_LIBPATH"] + os.pathsep + testEnvLibPath
 testEnvLibPath = testEnv.subst( testEnvLibPath )
 
-testEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] = testEnvLibPath
-if libraryPathEnvVar :
-	testEnv["ENV"][libraryPathEnvVar] = testEnvLibPath
-testEnv["ENV"]["IECORE_OP_PATHS"] = "test/IECore/ops"
+testEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] = os.pathsep.join( [ testEnv["ENV"].get(testEnv["TEST_LIBRARY_PATH_ENV_VAR"], ""), testEnvLibPath ] )
+if testEnv["TEST_LIBRARY_PATH_ENV_VAR"] != libraryPathEnvVar :
+	testEnv["ENV"][libraryPathEnvVar] = os.pathsep.join( [ testEnv["ENV"].get(libraryPathEnvVar, ""), testEnvLibPath ] )
+testEnv["ENV"]["IECORE_OP_PATHS"] = os.path.join( "test", "IECore", "ops" )
 
 c = Configure( testEnv )
 
 withBoostUnitTest = c.CheckLibWithHeader( env.subst( "boost_unit_test_framework" + env["BOOST_LIB_SUFFIX"] ), "boost/test/unit_test.hpp", "CXX" )
-withBoostTestExecMonitor = c.CheckLibWithHeader( env.subst( "boost_test_exec_monitor" + env["BOOST_LIB_SUFFIX"] ), "boost/test/test_exec_monitor.hpp", "CXX" )
+# Boost on Windows does includes the "lib" prefix for the boost_test_exec_monitor
+boostTestExecMonitorLibName = "libboost_test_exec_monitor" if testEnv["PLATFORM"] == "win32" else "boost_test_exec_monitor"
+withBoostTestExecMonitor = c.CheckLibWithHeader( env.subst( boostTestExecMonitorLibName + env["BOOST_LIB_SUFFIX"] ), "boost/test/test_exec_monitor.hpp", "CXX" )
 withBoostTest = withBoostUnitTest and withBoostTestExecMonitor
 
 c.Finish()
 
-testEnv["ENV"]["PYTHONPATH"] = "./python:" + testEnv.subst( "$PYTHONPATH" )
+testEnv["ENV"]["PYTHONPATH"] = os.pathsep.join(
+	[
+		os.path.join( ".", "python" ),
+		testEnv.subst( "$PYTHONPATH" )
+	]
+)
 
 if testEnv["PLATFORM"] == "darwin" :
 
@@ -1689,6 +1693,8 @@ def writePkgConfig( env, python_env ):
 
 coreEnv = env.Clone( IECORE_NAME="IECore" )
 coreEnv.Append( CXXFLAGS="-DIECore_EXPORTS" )
+if coreEnv["PLATFORM"] == "win32" : 
+	coreEnv.Append( LIBS="version.lib" )
 corePythonEnv = pythonEnv.Clone( IECORE_NAME="IECorePython" )
 corePythonEnv.Append( CXXFLAGS="-DIECorePython_EXPORTS" )
 corePythonModuleEnv = pythonModuleEnv.Clone( IECORE_NAME="IECore" )
@@ -1827,9 +1833,9 @@ imageEnvPrepends = {
 }
 
 imageEnv.Prepend( **imageEnvPrepends )
-# Windows does not have a default library path, it will find the needed libraries based on PATH environment variable
-if libraryPathEnvVar :
-	imageEnv["ENV"][libraryPathEnvVar] = imageEnv["LIBPATH"]
+# Windows uses PATH for to find libraries, we must append to it to make sure we don't overwrite existing PATH entries.
+# On Linux and MacOS this will append to an empty library path.
+imageEnv["ENV"][libraryPathEnvVar] = os.pathsep.join( [ imageEnv["ENV"].get(libraryPathEnvVar, "") ] + imageEnv["LIBPATH"] )
 
 if doConfigure :
 
@@ -1920,12 +1926,12 @@ if doConfigure :
 
 		# testing
 		imageTestEnv = testEnv.Clone()
-		imageTestEnv["ENV"]["PYTHONPATH"] = imageTestEnv["ENV"]["PYTHONPATH"] + ":python"
+		imageTestEnv["ENV"]["PYTHONPATH"] = os.pathsep.join( [ imageTestEnv["ENV"]["PYTHONPATH"], "python" ] )
 
-		imageEnvLibPath = ":".join( imageEnvPrepends["LIBPATH"] )
+		imageEnvLibPath = os.pathsep.join( imageEnvPrepends["LIBPATH"] )
 		imageLibs = imageTestEnv.subst( imageEnvLibPath )
 		imageTestLibs = imageTestEnv["ENV"][imageTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]]
-		imageTestEnv["ENV"][imageTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] = ":".join( [imageLibs, imageTestLibs] )
+		imageTestEnv["ENV"][imageTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] = os.pathsep.join( [imageLibs, imageTestLibs] )
 
 		imageTest = imageTestEnv.Command( "test/IECoreImage/results.txt", imagePythonModule, "$PYTHON $TEST_IMAGE_SCRIPT --verbose" )
 		NoCache( imageTest )
@@ -2097,7 +2103,7 @@ if doConfigure :
 		# testing
 		vdbTestEnv = testEnv.Clone()
 
-		vdbTestEnv["ENV"]["PYTHONPATH"] = vdbTestEnv["ENV"]["PYTHONPATH"] + ":" + vdbTestEnv["VDB_PYTHON_PATH"]
+		vdbTestEnv["ENV"]["PYTHONPATH"] = vdbTestEnv["ENV"]["PYTHONPATH"] + os.pathsep + vdbTestEnv["VDB_PYTHON_PATH"]
 
 		vdbTest = vdbTestEnv.Command( "test/IECoreVDB/results.txt", vdbPythonModule, "$PYTHON $TEST_VDB_SCRIPT --verbose" )
 		NoCache( vdbTest )
@@ -2473,9 +2479,9 @@ if doConfigure :
 
 		mayaTestEnv = testEnv.Clone()
 
-		mayaTestLibPaths = mayaEnv.subst( ":".join( mayaPythonModuleEnv["LIBPATH"] ) )
-		mayaTestEnv["ENV"][mayaTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += ":" + mayaTestLibPaths
-		mayaTestEnv["ENV"][libraryPathEnvVar] += ":" + mayaTestLibPaths
+		mayaTestLibPaths = mayaEnv.subst( os.pathsep.join( mayaPythonModuleEnv["LIBPATH"] ) )
+		mayaTestEnv["ENV"][mayaTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += os.pathsep + mayaTestLibPaths
+		mayaTestEnv["ENV"][libraryPathEnvVar] += os.pathsep + mayaTestLibPaths
 
 		mayaTestEnv["ENV"]["PATH"] = mayaEnv.subst( "$MAYA_ROOT/bin:" ) + mayaEnv["ENV"]["PATH"]
 		mayaTestEnv["ENV"]["MAYA_PLUG_IN_PATH"] = "./plugins/maya:./test/IECoreMaya/plugins"
@@ -2940,9 +2946,9 @@ if doConfigure :
 		#=====
 		houdiniTestEnv = testEnv.Clone()
 
-		houdiniTestLibPaths = houdiniEnv.subst( ":".join( houdiniPythonModuleEnv["LIBPATH"] ) )
-		houdiniTestEnv["ENV"][houdiniTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += ":" + houdiniTestLibPaths
-		houdiniTestEnv["ENV"][libraryPathEnvVar] += ":" + houdiniTestLibPaths
+		houdiniTestLibPaths = houdiniEnv.subst( os.pathsep.join( houdiniPythonModuleEnv["LIBPATH"] ) )
+		houdiniTestEnv["ENV"][houdiniTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += os.pathsep + houdiniTestLibPaths
+		houdiniTestEnv["ENV"][libraryPathEnvVar] += os.pathsep + houdiniTestLibPaths
 
 		houdiniTestEnv["ENV"]["PATH"] = houdiniEnv.subst( "$HOUDINI_ROOT/bin:" ) + houdiniEnv["ENV"]["PATH"]
 
@@ -3121,9 +3127,9 @@ if doConfigure :
 		# tests
 		arnoldTestEnv = testEnv.Clone()
 		arnoldTestEnv["ENV"]["PYTHONPATH"] += ":./contrib/IECoreArnold/python:" + arnoldEnv.subst( "$ARNOLD_ROOT/python" )
-		arnoldTestEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += ":" + arnoldEnv.subst( ":".join( arnoldPythonModuleEnv["LIBPATH"] ) )
-		arnoldTestEnv["ENV"]["PATH"] = arnoldEnv.subst( "$ARNOLD_ROOT/bin" ) + ":" + arnoldTestEnv["ENV"]["PATH"]
-		arnoldTestEnv["ENV"]["ARNOLD_PLUGIN_PATH"] = "contrib/IECoreArnold/test/IECoreArnold/plugins:contrib/IECoreArnold/test/IECoreArnold/metadata"
+		arnoldTestEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += os.pathsep + arnoldEnv.subst( os.pathsep.join( arnoldPythonModuleEnv["LIBPATH"] ) )
+		arnoldTestEnv["ENV"]["PATH"] = arnoldEnv.subst( "$ARNOLD_ROOT/bin" ) + os.pathsep + arnoldTestEnv["ENV"]["PATH"]
+		arnoldTestEnv["ENV"]["ARNOLD_PLUGIN_PATH"] = os.pathsep.join( [ "contrib/IECoreArnold/test/IECoreArnold/plugins", "contrib/IECoreArnold/test/IECoreArnold/metadata" ] )
 		arnoldTest = arnoldTestEnv.Command( "contrib/IECoreArnold/test/IECoreArnold/results.txt", arnoldPythonModule, "$PYTHON $TEST_ARNOLD_SCRIPT --verbose" )
 		NoCache( arnoldTest )
 		arnoldTestEnv.Depends( arnoldTest, [ arnoldPythonModule + arnoldDriverForTest + arnoldLibrary ] )
@@ -3285,8 +3291,8 @@ if doConfigure :
 		usdLibPath = coreEnv.subst("$USD_LIB_PATH")
 		usdPythonPath = os.path.join(usdLibPath, "python")
 
-		usdTestEnv["ENV"]["PYTHONPATH"] += ":" + usdPythonPath
-		usdTestEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += ":" + usdLibPath
+		usdTestEnv["ENV"]["PYTHONPATH"] += os.pathsep + usdPythonPath
+		usdTestEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += os.pathsep + usdLibPath
 
 		# setup pluginInfo for custom file format registration
 		testSdfPlugInfo = os.path.join( os.getcwd(), "plugins", "usd", "plugInfo.json" )
@@ -3429,8 +3435,8 @@ if doConfigure :
 
 		# tests
 		alembicTestEnv = testEnv.Clone()
-		alembicTestLibPaths = alembicEnv.subst( ":".join( alembicPythonModuleEnv["LIBPATH"] ) )
-		alembicTestEnv["ENV"][alembicTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += ":" + alembicTestLibPaths
+		alembicTestLibPaths = alembicEnv.subst( os.pathsep.join( alembicPythonModuleEnv["LIBPATH"] ) )
+		alembicTestEnv["ENV"][alembicTestEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += os.pathsep + alembicTestLibPaths
 		alembicTestEnv["ENV"]["PYTHONPATH"] += ":./contrib/IECoreAlembic/python"
 		alembicTest = alembicTestEnv.Command( "contrib/IECoreAlembic/test/IECoreAlembic/results.txt", alembicPythonModule, "$PYTHON $TEST_ALEMBIC_SCRIPT --verbose" )
 		NoCache( alembicTest )
@@ -3587,9 +3593,9 @@ if doConfigure :
 
 		# tests
 		appleseedTestEnv = testEnv.Clone()
-		appleseedTestEnv["ENV"]["PYTHONPATH"] += ":./contrib/IECoreAppleseed/python" + ":" + appleseedEnv.subst( "$APPLESEED_LIB_PATH/python" + pythonEnv["PYTHON_VERSION"] )
-		appleseedTestEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += ":" + appleseedEnv.subst( ":".join( appleseedPythonModuleEnv["LIBPATH"] ) )
-		appleseedTestEnv["ENV"]["PATH"] = appleseedEnv.subst( "$APPLESEED_ROOT/bin" ) + ":" + appleseedTestEnv["ENV"]["PATH"]
+		appleseedTestEnv["ENV"]["PYTHONPATH"] += ":./contrib/IECoreAppleseed/python" + os.pathsep + appleseedEnv.subst( "$APPLESEED_LIB_PATH/python" + pythonEnv["PYTHON_VERSION"] )
+		appleseedTestEnv["ENV"][testEnv["TEST_LIBRARY_PATH_ENV_VAR"]] += os.pathsep + appleseedEnv.subst( os.pathsep.join( appleseedPythonModuleEnv["LIBPATH"] ) )
+		appleseedTestEnv["ENV"]["PATH"] = appleseedEnv.subst( "$APPLESEED_ROOT/bin" ) + os.pathsep + appleseedTestEnv["ENV"]["PATH"]
 		appleseedTestEnv["ENV"]["APPLESEED_SEARCHPATH"] = os.getcwd() + "/contrib/IECoreAppleseed/test/IECoreAppleseed/plugins"
 		appleseedTest = appleseedTestEnv.Command( "contrib/IECoreAppleseed/test/IECoreAppleseed/results.txt", appleseedPythonModule, "$PYTHON $TEST_APPLESEED_SCRIPT --verbose" )
 		NoCache( appleseedTest )

config/azure/build.yaml

@@ -60,11 +60,11 @@ jobs:
 
    - script: |
        ${{ parameters.envSetupCmd }}
-       scons -j ${{ parameters.threads }} install BUILD_TYPE=${{ parameters.buildType }} OPTIONS=${{ parameters.options }}
+       scons -j ${{ parameters.threads }} install BUILD_TYPE=${{ parameters.buildType }} OPTIONS=${{ parameters.options }} CXXSTD=c++14
      displayName: 'Build'
 
    - script: |
-       scons testCore testCorePython testScene testImage testAlembic testAppleseed OPTIONS=${{ parameters.options }}
+       scons testCore testCorePython testScene testImage testAlembic testAppleseed OPTIONS=${{ parameters.options }} CXXSTD=c++14
        cat test/IECore/results.txt
      displayName: 'Test'
      condition: and( succeeded(), eq( variables['Agent.OS'], 'Linux' ) )