py modules: defer linking and symbols to final py interp (#475)

* py modules: defer linking and symbols to final py interp

* add  undef dynamic_lookup logic 

* cmake change: avoid NATIVE_COMMAND in separate_arguments  to support older vers of cmake

* appveyor: skip python 2.7 testing b/c default miniconda numpy is broken

appveyor.yml

@@ -1,19 +1,29 @@
+image:
+  - Visual Studio 2017
 environment:
   matrix:
-  - CMAKE_GENERATOR: "Visual Studio 14 2015"
+  # - CMAKE_GENERATOR: "Visual Studio 15 2017"
+  #   CONFIG: Release
+  #   PYTHON_VERSION: 2.7
+  #   BUILD_SHARED_LIBS: ON
+  #   ENABLE_PYTHON: ON
+  #   MINICONDA: C:\Miniconda
+  #   ENABLE_MPI: ON
+  #   MPI_HOME: "C:/Program Files (x86)/Microsoft SDKs/MPI"
+  - CMAKE_GENERATOR: "Visual Studio 15 2017"
     CONFIG: Release
-    PYTHON_VERSION: 2.7
+    PYTHON_VERSION: 3.6
     BUILD_SHARED_LIBS: ON
     ENABLE_PYTHON: ON
-    MINICONDA: C:\Miniconda
+    MINICONDA: C:\Miniconda36
     ENABLE_MPI: ON
     MPI_HOME: "C:/Program Files (x86)/Microsoft SDKs/MPI"
-  - CMAKE_GENERATOR: "Visual Studio 14 2015"
+  - CMAKE_GENERATOR: "Visual Studio 15 2017"
     CONFIG: Release
-    PYTHON_VERSION: 2.7
+    PYTHON_VERSION: 3.6
     BUILD_SHARED_LIBS: OFF
     ENABLE_PYTHON: OFF
-    MINICONDA: C:\Miniconda
+    MINICONDA: C:\Miniconda36
     ENABLE_MPI: ON
     MPI_HOME: "C:/Program Files (x86)/Microsoft SDKs/MPI"
 
@@ -29,10 +39,23 @@ install:
   #- 7z x hdf5.zip -y
   # setup python via mini conda
   #- ps: Start-Process -FilePath msiexec -ArgumentList /i, "hdf5\HDF5-1.8.18-win64.msi", /quiet -Wait
+  #
+  ####
+  # remove default python paths to avoid crossing with miniconda
+  ####
+  - set PATH=%PATH:C:\Python27;=%
+  - set PATH=%PATH:C:\Python27\Scripts;=%
+  # add built-in mini conda path
   - "set PATH=%MINICONDA%;%MINICONDA%\\Scripts;%PATH%"
-  - conda update --yes conda
-  - conda install --yes numpy
-
+  # check path
+  - "echo %PATH%"
+  - "echo %PYTHONPATH%"
+  - conda config --set always_yes yes --set changeps1 no
+  - conda update -q conda
+  - conda info -a
+  - conda install numpy
+  # try to import numpy to as conda gut-check
+  - python -c "import numpy as n; print(n.__version__); print(n.get_include());"
   # Install MS-MPI
   - ps: Start-FileDownload 'https://download.microsoft.com/download/B/2/E/B2EB83FE-98C2-4156-834A-E1711E6884FB/MSMpiSetup.exe'
   - MSMpiSetup.exe -unattend

src/cmake/thirdparty/FindNumPy.cmake

@@ -120,6 +120,9 @@ if(PYTHONINTERP_FOUND)
         if(NumPy_FIND_REQUIRED)
             message(FATAL_ERROR
                 "NumPy import failure:\n${_NUMPY_ERROR_VALUE}")
+        else()
+            message(STATUS
+                "NumPy import failure:\n${_NUMPY_ERROR_VALUE}")
         endif()
         set(NUMPY_FOUND FALSE)
     endif()

src/cmake/thirdparty/SetupHDF5.cmake

@@ -226,8 +226,18 @@ string(SUBSTRING "${hdf5_tpl_lnk_libs}" 0 ${hdf5_tpl_lnk_libs_end_pos} hdf5_tpl_
 if(${hdf5_tpl_lnk_libs})
     string(STRIP "${hdf5_tpl_lnk_libs}" hdf5_tpl_lnk_libs)
 endif()
+
 # add -l to any libraries that are just their names (like "m" instead of "-lm")
-separate_arguments(_temp_link_libs NATIVE_COMMAND ${hdf5_tpl_lnk_libs})
+# ** Note **
+# The NATIVE_COMMAND arg to separate_arguments() was added in CMake 3.9
+# instead use strategy that allows older versions of CMake:
+#    an if to select WINDOWS_COMMAND or UNIX_COMMAND arg
+if(WIN32)
+    separate_arguments(_temp_link_libs WINDOWS_COMMAND ${hdf5_tpl_lnk_libs})
+else()
+    separate_arguments(_temp_link_libs UNIX_COMMAND ${hdf5_tpl_lnk_libs})
+endif()
+
 set(_fixed_link_libs)
 foreach(lib ${_temp_link_libs})
     if(NOT "${lib}" MATCHES ^[-/])

src/cmake/thirdparty/SetupPython.cmake

@@ -69,7 +69,24 @@ if(PYTHONINTERP_FOUND)
                         OUTPUT_VARIABLE PYTHON_LIB_DIR
                         ERROR_VARIABLE ERROR_FINDING_LIB_DIR)
         MESSAGE(STATUS "PYTHON_LIB_DIR ${PYTHON_LIB_DIR}")
-
+
+
+        # check if we need "-undefined dynamic_lookup" by inspecting LDSHARED flags
+        execute_process(COMMAND "${PYTHON_EXECUTABLE}" "-c" 
+                                "import sys;import sysconfig;sys.stdout.write(sysconfig.get_config_var('LDSHARED'))"
+                        OUTPUT_VARIABLE PYTHON_LDSHARED_FLAGS
+                        ERROR_VARIABLE ERROR_FINDING_PYTHON_LDSHARED_FLAGS)
+
+        MESSAGE(STATUS "PYTHON_LDSHARED_FLAGS ${PYTHON_LDSHARED_FLAGS}")
+
+        if(PYTHON_LDSHARED_FLAGS MATCHES "-undefined dynamic_lookup")
+             MESSAGE(STATUS "PYTHON_USE_UNDEFINED_DYNAMIC_LOOKUP_FLAG is ON")
+            set(PYTHON_USE_UNDEFINED_DYNAMIC_LOOKUP_FLAG ON)
+        else()
+             MESSAGE(STATUS "PYTHON_USE_UNDEFINED_DYNAMIC_LOOKUP_FLAG is OFF")
+            set(PYTHON_USE_UNDEFINED_DYNAMIC_LOOKUP_FLAG OFF)
+        endif()
+
         # check for python libs differs for windows python installs
         if(NOT WIN32)
             # use shared python if we are using shared libs
@@ -236,8 +253,21 @@ FUNCTION(PYTHON_ADD_HYBRID_MODULE target_name
     MESSAGE(STATUS "${target_name} build location: ${CMAKE_BINARY_DIR}/${dest_dir}/${py_module_dir}")
 
 
-    # link with python
-    target_link_libraries(${target_name} ${PYTHON_LIBRARIES})
+    # macOS and linux
+    # defer linking with python, let the final python interpreter
+    # provide the proper symbols
+    #
+    # on osx we may need to use the following flag to 
+    # avoid linking errors
+    if(PYTHON_USE_UNDEFINED_DYNAMIC_LOOKUP_FLAG)
+        set_target_properties(${target_name} PROPERTIES 
+                              LINK_FLAGS "-undefined dynamic_lookup")
+    endif()
+
+    # win32, link to python
+    if(WIN32)
+        target_link_libraries(${target_name} ${PYTHON_LIBRARIES})
+    endif()
 
     # support installing the python module components to an
     # an alternate dir, set via PYTHON_MODULE_INSTALL_PREFIX 

src/libs/blueprint/python/conduit_blueprint_mcarray_examples_python.cpp

@@ -326,9 +326,9 @@ static struct PyModuleDef blueprint_mcarray_examples_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_BLUEPRINT_PYTHON_API PyInit_conduit_blueprint_mcarray_examples_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API PyObject *PyInit_conduit_blueprint_mcarray_examples_python(void)
 #else
-void CONDUIT_BLUEPRINT_PYTHON_API initconduit_blueprint_mcarray_examples_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API void initconduit_blueprint_mcarray_examples_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {    

src/libs/blueprint/python/conduit_blueprint_mcarray_python.cpp

@@ -530,9 +530,9 @@ static struct PyModuleDef blueprint_mcarray_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_BLUEPRINT_PYTHON_API PyInit_conduit_blueprint_mcarray_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API PyObject *PyInit_conduit_blueprint_mcarray_python(void)
 #else
-void CONDUIT_BLUEPRINT_PYTHON_API initconduit_blueprint_mcarray_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API void initconduit_blueprint_mcarray_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {    

src/libs/blueprint/python/conduit_blueprint_mesh_examples_python.cpp

@@ -581,9 +581,9 @@ static struct PyModuleDef blueprint_mesh_examples_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_BLUEPRINT_PYTHON_API PyInit_conduit_blueprint_mesh_examples_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API PyObject *PyInit_conduit_blueprint_mesh_examples_python(void)
 #else
-void CONDUIT_BLUEPRINT_PYTHON_API initconduit_blueprint_mesh_examples_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API void initconduit_blueprint_mesh_examples_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {    

src/libs/blueprint/python/conduit_blueprint_mesh_python.cpp

@@ -423,9 +423,9 @@ static struct PyModuleDef blueprint_mesh_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_BLUEPRINT_PYTHON_API PyInit_conduit_blueprint_mesh_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API PyObject *PyInit_conduit_blueprint_mesh_python(void)
 #else
-void CONDUIT_BLUEPRINT_PYTHON_API initconduit_blueprint_mesh_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API void initconduit_blueprint_mesh_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {    

src/libs/blueprint/python/conduit_blueprint_python.cpp

@@ -354,9 +354,9 @@ static struct PyModuleDef blueprint_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_BLUEPRINT_PYTHON_API PyInit_conduit_blueprint_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API PyObject * PyInit_conduit_blueprint_python(void)
 #else
-void CONDUIT_BLUEPRINT_PYTHON_API initconduit_blueprint_python(void)
+CONDUIT_BLUEPRINT_PYTHON_API void initconduit_blueprint_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {    

src/libs/conduit/python/conduit_python.cpp

@@ -5873,7 +5873,7 @@ static struct PyModuleDef conduit_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_PYTHON_API PyInit_conduit_python(void)
+CONDUIT_PYTHON_API PyObject * PyInit_conduit_python(void)
 #else
 void CONDUIT_PYTHON_API initconduit_python(void)
 #endif

src/libs/relay/python/conduit_relay_io_python.cpp

@@ -1088,9 +1088,9 @@ static struct PyModuleDef relay_io_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_RELAY_PYTHON_API PyInit_conduit_relay_io_python(void)
+CONDUIT_RELAY_PYTHON_API PyObject * PyInit_conduit_relay_io_python(void)
 #else
-void CONDUIT_RELAY_PYTHON_API initconduit_relay_io_python(void)
+CONDUIT_RELAY_PYTHON_API void initconduit_relay_io_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {    

src/libs/relay/python/conduit_relay_python.cpp

@@ -272,9 +272,9 @@ static struct PyModuleDef relay_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_RELAY_PYTHON_API PyInit_conduit_relay_python(void)
+CONDUIT_RELAY_PYTHON_API PyObject * PyInit_conduit_relay_python(void)
 #else
-void CONDUIT_RELAY_PYTHON_API initconduit_relay_python(void)
+CONDUIT_RELAY_PYTHON_API void initconduit_relay_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {    

src/libs/relay/python/conduit_relay_web_python.cpp

@@ -984,9 +984,9 @@ static struct PyModuleDef relay_web_python_module_def =
 extern "C" 
 //---------------------------------------------------------------------------//
 #if defined(IS_PY3K)
-PyObject *CONDUIT_RELAY_PYTHON_API PyInit_conduit_relay_web_python(void)
+CONDUIT_RELAY_PYTHON_API PyObject * PyInit_conduit_relay_web_python(void)
 #else
-void CONDUIT_RELAY_PYTHON_API initconduit_relay_web_python(void)
+CONDUIT_RELAY_PYTHON_API void initconduit_relay_web_python(void)
 #endif
 //---------------------------------------------------------------------------//
 {