Merge d47bb66 into a03a273

.travis.yml

@@ -2,16 +2,48 @@
 
 dist: trusty
 
+env:
+    global:
+        - GIT_DESCRIBE_TAG=0.29.2
+        - GIT_DESCRIBE_NUMBER=1
+        - CONDA_ENV=travisci
+        - PYTHON_VER=3.7
+        - NUMPY_VER=1.16
+    matrix:
+        - HPAT_CHECK_STYLE=True
+        - HPAT_RUN_BUILD=True HPAT_WHEELS=True
+        - HPAT_RUN_BUILD=True HPAT_RUN_COVERAGE=True
+        - HPAT_RUN_BUILD=True HPAT_CONFIG_MPI=False
 matrix:
-    include: 
-        - env: CONDA_ENV=travisci PYTHON_VER=3.7
+    allow_failures:
+        - env: HPAT_CHECK_STYLE=True
+        - env: HPAT_RUN_BUILD=True HPAT_CONFIG_MPI=False
 
-# TODO Remove travis from repository and add code coverage check
-before_install: skip
+before_install:
+    - buildscripts/setup_conda.sh
+    - export PATH=$HOME/miniconda3/bin:$PATH
+    - if [ "$HPAT_CHECK_STYLE" == "True" ]; then
+        sudo apt-get update -y;
+        sudo apt-get install -y clang-format-6.0;
+      fi
 
-install: skip
+install:
+    - export PATH=$HOME/miniconda3/bin:$PATH
+    - export HPAT_SOURCE_DIR="$(pwd)/hpat"
+    - env | grep HPAT_SOURCE_DIR
+    - export HPAT_WHEELS_DIR="$(pwd)/hpat-wheels"
+    - env | grep HPAT_WHEELS_DIR
+    - mkdir $HPAT_WHEELS_DIR
+    - if [ "$HPAT_RUN_BUILD" == "True" ]; then
+          buildscripts/build.sh;
+      fi
 
-script: skip
+script:
+    - export PATH=$HOME/miniconda3/bin:$PATH
+    - if [ "$HPAT_CHECK_STYLE" == "True" ]; then
+          chmod 777 buildscripts/code_style.sh;
+          buildscripts/code_style.sh;
+      fi
 
 after_success: skip
 

azure-pipelines.yml

@@ -1,64 +1,40 @@
 jobs:
-- job: Windows
-  timeoutInMinutes: 0
-  pool:
-    vmImage: 'vs2017-win2016'
-
-  steps:
-  - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
-    displayName: Add conda to PATH
-
-  - script: conda create -q -y -n CB python=3.7 conda-build conda-verify vc vs2015_runtime vs2015_win-64
-    displayName: Create Anaconda environment
+- template: buildscripts/azure-template-windows.yml
+  parameters:
+    name: Windows
+    vmImage: vs2017-win2016
+    matrix:
+      py37_np116_numpes2:
+        CONDA_ENV: 'travisci'
+        HPAT_NUM_PES: '2'
+        PYTHON_VER: '3.7'
+        NUMPY_VER: '1.16'
 
-  - script: |
-      call activate CB
-      conda build --python 3.7 --override-channels -c numba -c conda-forge -c defaults -c intel --output-folder=. buildscripts/hpat-conda-recipe/
-    displayName: conda build
+      py37_np116_wheel_numpes2:
+        CONDA_ENV: 'travisci'
+        HPAT_NUM_PES: '2'
+        PYTHON_VER: '3.7'
+        NUMPY_VER: '1.16'
+        HPAT_WHEELS: 'True'
 
 - template: buildscripts/azure-template-linux-macos.yml
   parameters:
     name: macOS
     vmImage: xcode9-macos10.13
     matrix:
-      py37_numpes0:
-        CONDA_ENV: 'travisci'
-        PYTHON_VER: '3.7'
-
-      py37_numpes2:
+      py37_np116_numpes2:
         CONDA_ENV: 'travisci'
         HPAT_NUM_PES: '2'
         PYTHON_VER: '3.7'
+        NUMPY_VER: '1.16'
 
 - template: buildscripts/azure-template-linux-macos.yml
   parameters:
     name: Linux
     vmImage: ubuntu-16.04
     matrix:
-      py36_numpes0:
-        CONDA_ENV: 'travisci'
-        PYTHON_VER: '3.6'
-
-      py36_numpes3:
-        CONDA_ENV: 'travisci'
-        HPAT_NUM_PES: '3'
-        PYTHON_VER: '3.6'
-
-      py37_numpes0:
-        CONDA_ENV: 'travisci'
-        PYTHON_VER: '3.7'
-
-      py37_numpes1:
-        CONDA_ENV: 'travisci'
-        HPAT_NUM_PES: '1'
-        PYTHON_VER: '3.7'
-
-      py37_numpes2:
+      py37_np116_numpes2:
         CONDA_ENV: 'travisci'
         HPAT_NUM_PES: '2'
         PYTHON_VER: '3.7'
-
-      py37_numpes3:
-        CONDA_ENV: 'travisci'
-        HPAT_NUM_PES: '3'
-        PYTHON_VER: '3.7'
+        NUMPY_VER: '1.16'

buildscripts/azure-template-linux-macos.yml

@@ -5,7 +5,7 @@ parameters:
 
 jobs:
 - job: ${{ parameters.name }}
-  timeoutInMinutes: 240
+  timeoutInMinutes: 120
   pool:
     vmImage: ${{ parameters.vmImage }}
   strategy:

buildscripts/azure-template-windows.yml

@@ -0,0 +1,29 @@
+parameters:
+  name: ''
+  vmImage: ''
+  matrix: []
+
+jobs:
+- job: ${{ parameters.name }}
+  timeoutInMinutes: 120
+  pool: 
+    vmImage: ${{ parameters.vmImage }}
+  strategy:
+    maxParallel: 11
+    matrix:
+      ${{ insert }}: ${{ parameters.matrix }}
+
+  steps:
+    - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
+      displayName: Add conda to PATH
+
+    - script: conda create -q -y -n %CONDA_ENV% python=%PYTHON_VER% conda-build conda-verify vc vs2015_runtime vs2015_win-64
+      displayName: Create Anaconda environment
+
+    - script: |
+        call activate %CONDA_ENV%
+        %HPAT_WHEELS_DIR% = %~dp0\hpat-wheels
+        echo %HPAT_WHEELS_DIR%
+        mkdir %HPAT_WHEELS_DIR%
+        conda build --python %PYTHON_VER% --numpy=%NUMPY_VER% --override-channels -c numba -c conda-forge -c defaults -c intel buildscripts/hpat-conda-recipe/
+      displayName: conda build

buildscripts/build.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
+set -ex
 
 source activate $CONDA_ENV
 
-conda build --python $PYTHON_VER -c numba -c conda-forge -c defaults --override-channels ./buildscripts/hpat-conda-recipe/
+conda build --python $PYTHON_VER --numpy=$NUMPY_VER -c numba -c conda-forge -c defaults --override-channels ./buildscripts/hpat-conda-recipe/

buildscripts/code_style.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+source activate $CONDA_ENV
+
+echo "======================================================================"
+echo Executing clang-format-6.0 style:
+echo "======================================================================"
+python ./setup.py style
+
+echo "======================================================================"
+echo Executing python flake8:
+echo "======================================================================"
+flake8 ./

buildscripts/hpat-conda-recipe/bld.bat

@@ -1,11 +1,14 @@
+echo on
+
 set HDF5_DIR="%LIBRARY_PREFIX%"
-REM set OPENCV_DIR="%LIBRARY_PREFIX%"
-REM set DAALROOT="%LIBRARY_PREFIX%"
 "%PYTHON%" setup.py build install --single-version-externally-managed --record=record.txt
 if errorlevel 1 exit 1
 
-
-REM "%PYTHON%" setup.py build_doc
-REM if errorlevel 1 exit 1
-REM "%PYTHON%" setup.py build_devdoc
-REM if errorlevel 1 exit 1
+@rem Build HPAT wheel
+echo Build HPAT wheel
+IF "%HPAT_WHEELS%" == "True" (
+    "%PYTHON%" setup.py bdist_wheel
+    if errorlevel 1 exit 1
+    copy dist\hpat*.whl "%HPAT_WHEELS_DIR%"
+    if errorlevel 1 exit 1
+    )

buildscripts/hpat-conda-recipe/build.sh

@@ -1,9 +1,15 @@
-# LDSHARED="mpicxx -cxx=$GXX -shared" LD="mpicxx -cxx=$GXX" \
-# CC="mpicxx -cxx=$GXX -std=c++11" GXX="mpicxx -cxx=$GXX -std=c++11" \
-# OPENCV_DIR="${PREFIX}" DAALROOT="${PREFIX}"
+set -ex
+
 HDF5_DIR="${PREFIX}" MACOSX_DEPLOYMENT_TARGET=10.9 \
 $PYTHON setup.py build install --single-version-externally-managed --record=record.txt
 
-#Build Documentation
-#$PYTHON setup.py build_doc
-#$PYTHON setup.py build_devdoc
+if [ "$HPAT_WHEELS" == "True" ]; then
+  if [ -z "$HPAT_WHEELS_DIR" ]; then
+    echo "Please set HPAT_WHEELS_DIR to build HPAT wheels"
+  else
+    # Build HPAT wheel
+    echo Build HPAT wheel
+    $PYTHON setup.py bdist_wheel
+    cp dist/hpat*.whl "$HPAT_WHEELS_DIR"
+  fi
+fi

buildscripts/hpat-conda-recipe/meta.yaml

@@ -11,6 +11,9 @@ build:
     - HPAT_CONFIG_MPI
     - HPAT_NUM_PES
     - HPAT_RUN_COVERAGE
+    - HPAT_SOURCE_DIR
+    - HPAT_WHEELS
+    - HPAT_WHEELS_DIR
 
 requirements:
   build:
@@ -32,6 +35,7 @@ requirements:
     - impi_rt # [win]
     - impi-devel # [win]
     - conda-package-handling ==1.3.11 #[win]
+    - wheel
 
   run:
     - python
@@ -50,6 +54,7 @@ test:
     - h5py
     - scipy
     - coveralls
+    - pip
   imports:
     - hpat
 

buildscripts/hpat-conda-recipe/run_test.bat

@@ -1,17 +1,25 @@
+echo on
+
 set NUMBA_DEVELOPER_MODE=1
 set NUMBA_DISABLE_ERROR_MESSAGE_HIGHLIGHTING=1
 set PYTHONFAULTHANDLER=1
 
-mpiexec -localonly -n 1 python -m hpat.tests.gen_test_data
-if errorlevel 1 exit 1
-
-mpiexec -localonly -n 1 python -u -m hpat.runtests -v
+python -m hpat.tests.gen_test_data
 if errorlevel 1 exit 1
 
-mpiexec -localonly -n 2 python -u -m hpat.runtests -v
+IF "%HPAT_WHEELS%" == "True" (
+    conda remove -y hpat
+    pip install %HPAT_WHEELS_DIR%\hpat*.whl
+    python -c "import hpat"
+    )
 if errorlevel 1 exit 1
 
-mpiexec -localonly -n 3 python -u -m hpat.runtests -v
+@rem TODO investigate root cause of NumbaPerformanceWarning
+@rem http://numba.pydata.org/numba-doc/latest/user/parallel.html#diagnostics
+IF "%HPAT_NUM_PES%" == "" (
+    python -W ignore -u -m hpat.runtests -v
+    ) ELSE (
+    mpiexec -localonly -n %HPAT_NUM_PES% python -W ignore -u -m hpat.runtests -v)
 if errorlevel 1 exit 1
 
 REM Link check for Documentation using Sphinx's in-built linkchecker

buildscripts/hpat-conda-recipe/run_test.sh

@@ -1,20 +1,32 @@
 #!/bin/bash
 
-set -e
+set -ex
 
 export NUMBA_DEVELOPER_MODE=1
 export NUMBA_DISABLE_ERROR_MESSAGE_HIGHLIGHTING=1
 export PYTHONFAULTHANDLER=1
 
 python -m hpat.tests.gen_test_data
 
-#Link check for Documentation using Sphinx's in-built linkchecker
-#sphinx-build -b linkcheck -j1 usersource _build/html
+if [ "$HPAT_WHEELS" == "True" ]; then
+  conda remove -y hpat
+  pip install ${HPAT_WHEELS_DIR}/hpat*.whl
+  python -c "import hpat"
+fi
 
-# TODO investigate root cause of NumbaPerformanceWarning
-# http://numba.pydata.org/numba-doc/latest/user/parallel.html#diagnostics
-if [ -z "$HPAT_NUM_PES" ]; then
-  python -W ignore -u -m hpat.runtests -v
+if [ "$HPAT_RUN_COVERAGE" == "True" ]; then
+  coverage erase
+  coverage run --source=${HPAT_SOURCE_DIR} --omit ${HPAT_SOURCE_DIR}/ml/*,${HPAT_SOURCE_DIR}/xenon_ext.py,${HPAT_SOURCE_DIR}/ros.py,${HPAT_SOURCE_DIR}/cv_ext.py,${HPAT_SOURCE_DIR}/tests/* ${HPAT_SOURCE_DIR}/runtests.py
+  coveralls -v
 else
-  mpiexec -n $HPAT_NUM_PES python -W ignore -u -m hpat.runtests -v
+  #Link check for Documentation using Sphinx's in-built linkchecker
+  #sphinx-build -b linkcheck -j1 usersource _build/html
+
+  # TODO investigate root cause of NumbaPerformanceWarning
+  # http://numba.pydata.org/numba-doc/latest/user/parallel.html#diagnostics
+  if [ -z "$HPAT_NUM_PES" ]; then
+    python -W ignore -u -m hpat.runtests -v
+  else
+    mpiexec -n $HPAT_NUM_PES python -W ignore -u -m hpat.runtests -v
+  fi
 fi

buildscripts/setup_conda.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -ex
 
 # Install Miniconda
 # Reference:
@@ -22,5 +23,8 @@ export PATH=$HOME/miniconda3/bin:$PATH
 # Create conda env
 conda create -n $CONDA_ENV -q -y python=$PYTHON_VER
 source activate $CONDA_ENV
-# Install conda-build
-$CONDA_INSTALL conda-build
+# Install conda-build and numpy with pycodestyle (required for style check)
+$CONDA_INSTALL conda-build -c conda-forge
+if [ "$HPAT_CHECK_STYLE" == "True" ]; then
+  $CONDA_INSTALL numpy pycodestyle flake8 -c conda-forge
+fi