Merge pull request #37 from CDAT/new_conda_build_way1

use conda-recipes' build_tools/conda_build.py

.circleci/config.yml

@@ -1,178 +1,233 @@
 version: 2
 
-checkout:
-  post:
-    - ./ci-support/checkout_merge_commit.sh
-
 aliases:
   - &setup_miniconda
     name: setup_miniconda
     command: |
-      mkdir -p workspace
-      git clone -b validateNightly git@github.com:CDAT/cdat workspace/cdat
-      python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
+       mkdir -p workspace
+       git clone -b validateNightly git@github.com:CDAT/cdat workspace/cdat
+       # install_miniconda.py installs miniconda3 under $WORKDIR/miniconda
+       python workspace/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
 
-  - &create_conda_env
-    name: create_conda_env
-    environment:
-       CHANNELS: "-c cdat/label/nightly -c conda-forge -c cdat"
-       PKGS: "nbformat testsrunner"
+  - &conda_rerender
+    name: conda_rerender
     command: |
-       export PATH=$WORKDIR/miniconda/bin:$PATH
-       conda config --set always_yes yes --set changeps1 no
-       conda update -y -q conda
-       conda config --set anaconda_upload no
-       if [[ $PY_VER = "py2" ]]; then
-          conda create -q -n $PY_VER $CHANNELS $PKGS python=2
-       else
-          conda create -q -n $PY_VER $CHANNELS $PKGS "python>=3.7,<3.8" $COVERAGE_PKGS
-       fi
+       git clone https://github.com/CDAT/conda-recipes.git $WORKDIR/conda-recipes 
+       source $WORKDIR/miniconda/etc/profile.d/conda.sh
+       conda activate base
+       BUILD_SCRIPT="$WORKDIR/conda-recipes/build_tools/conda_build.py"
+       python $BUILD_SCRIPT -w $WORKDIR -l $LAST_STABLE -B 0 -p $PKG_NAME -b $CIRCLE_BRANCH --do_rerender
 
-  - &setup_cdat_info
-    name: setup_cdat_info
+  - &conda_build
+    name: conda_build
     command: |
-       export PATH=$WORKDIR/miniconda/bin:$PATH
-       export CDAT_ANONYMOUS_LOG=False
-       source activate $PY_VER
-       python setup.py install
+       source $WORKDIR/miniconda/etc/profile.d/conda.sh
+       conda activate base
+       conda config --add channels conda-forge --force
+       conda config --add channels cdat/label/nightly --force
+       conda config --set channel_priority strict
+       BUILD_SCRIPT="$WORKDIR/conda-recipes/build_tools/conda_build.py"
+       python $BUILD_SCRIPT -w $WORKDIR -p $PKG_NAME --build_version $BUILD_VARIANT_VER --do_build
 
+  - &setup_run_tests
+    name: setup_run_tests
+    environment:
+       PKGS: "nbformat testsrunner"
+       CHANNELS: "-c cdat/label/nightly -c conda-forge"
+    command: |
+       source $WORKDIR/miniconda/etc/profile.d/conda.sh
+       conda activate base
+       conda create -y -n $ENV_NAME --use-local $CHANNELS "$CONDA_PY_VER" $PKG_NAME $PKGS $COVERAGE_PKGS
+       conda activate $ENV_NAME
+       conda list
 
   - &run_tests
     name: run_tests
     command: |
-       export PATH=$WORKDIR/miniconda/bin:$PATH
-       export CDAT_ANONYMOUS_LOG=False
-       export CDAT_SETUP_PATH=${HOME}
-       source activate $PY_VER
+       source $WORKDIR/miniconda/etc/profile.d/conda.sh
+       conda activate $ENV_NAME
+       # NOTE: we need to do setup.py install for cdat_info (but not necessarily for other projects)
+       # one reason was due to cdat_info/__init__.py writes out a file that is accessed when
+       # we import the package
+       # DO NOT just copy 
+       python setup.py install
+       python -c "import $PKG_NAME"
        python run_tests.py -H -v2 -n 2 $COVERAGE
-       RESULT=$?
-       echo "*** $PY_VER test result: "${RESULT} 
-       exit $RESULT
+       conda deactivate
 
   - &conda_upload
     name: conda_upload
-    environment:
-       CHANNELS: "-c cdat/label/nightly -c conda-forge -c cdat"
     command: |
-       if [[ $CIRCLE_BRANCH != 'master' ]]; then
+       if [[ $CIRCLE_BRANCH != "master" ]]; then
           exit 0
        fi
-       export PATH=${HOME}/project/$WORKDIR/miniconda/bin:$PATH
-       conda install conda-build anaconda-client
-       conda config --set anaconda_upload no
-       export CONDA_BLD_PATH=`pwd`/conda_build
-       git clone git://github.com/cdat/conda-recipes
-       cd conda-recipes
-       if [[ -d vcs ]]; then
-          rm -rf cdat_info
-       fi
-       ln -s ../recipe cdat_info
-       export PKG_NAME=cdat_info
-       export USER=cdat
-       export VERSION=8.2
-       export LABEL=nightly
-       python ./prep_for_build.py -l $VERSION -b $CIRCLE_BRANCH
-       conda build $PKG_NAME $CHANNELS --python=3.7
-       anaconda -t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL $CONDA_BLD_PATH/noarch/$PKG_NAME-$VERSION.`date +%Y*`0.tar.bz2 --force
+       source $WORKDIR/miniconda/etc/profile.d/conda.sh
+       conda activate base
+       UPLOAD_OPTIONS="-t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL"
+       anaconda $UPLOAD_OPTIONS $WORKDIR/miniconda/conda-bld/noarch/$PKG_NAME-$VERSION.`date +%Y*`0.tar.bz2 --force
 
   - &run_coveralls
     name: run_coveralls
     command: |
-       export PATH=$WORKDIR/miniconda/bin:$PATH
-       source activate $PY_VER
+       source $WORKDIR/miniconda/etc/profile.d/conda.sh
+       conda activate $ENV_NAME
        coveralls
-       source deactivate
+       conda deactivate
 
 jobs:
-  macos_cdat_info_py2:
-    macos:
-      xcode: "11.1.0"
-    environment:
-      WORKDIR: "workspace/test_macos_cdat_info_py2"
-      OS: "osx-64"
-      PY_VER: "py2"
-    steps:
-      - checkout
-      - run: *setup_miniconda
-      - run: *create_conda_env
-      - run: *setup_cdat_info
-      - run: *run_tests
-      - store_artifacts:
-          path: tests_html
-          destination: tests_html
-      - store_artifacts:
-          path: tests_png
-          destination: tests_png
+   macos_setup:
+      macos:
+         xcode: "11.3.0"
+      environment:
+         WORKDIR: /Users/distiller/project/macos_build
+         PKG_NAME: "cdat_info"
+         REPO_NAME: "cdat_info"
+         LAST_STABLE: "8.2"
+         BUILD_VARIANT_VER: "noarch"
+      steps:
+         - checkout
+         - run: *setup_miniconda
+         - run: *conda_rerender
+         - run: *conda_build
+         - persist_to_workspace:
+              root: .
+              paths: 
+                 - macos_build
 
-  macos_cdat_info_py3:
-    macos:
-      xcode: "11.1.0"
-    environment:
-      WORKDIR: "workspace/test_macos_cdat_info_py3"
-      OS: "osx-64"
-      PY_VER: "py3"
-    steps:
-      - checkout
-      - run: *setup_miniconda
-      - run: *create_conda_env
-      - run: *setup_cdat_info
-      - run: *run_tests
-      - store_artifacts:
-          path: tests_html
-          destination: tests_html
-      - store_artifacts:
-          path: tests_png
-          destination: tests_png
+   linux_setup:
+      machine:
+         image: circleci/classic:latest
+      environment:
+         WORKDIR: /home/circleci/project/linux_build
+         PKG_NAME: "cdat_info"
+         REPO_NAME: "cdat_info"
+         LAST_STABLE: "8.2"
+         BUILD_VARIANT_VER: "noarch"
+      steps:
+         - checkout
+         - run: *setup_miniconda
+         - run: *conda_rerender
+         - run: *conda_build
+         - persist_to_workspace:
+              root: .
+              paths:
+                 - linux_build
 
-  linux_cdat_info_py2:
-    machine:
-      image: circleci/classic:latest
-    environment:
-      WORKDIR: "workspace/test_linux_cdat_info_py2"
-      OS: "linux-64"
-      PY_VER: "py2"
-    steps:
-      - checkout
-      - run: *setup_miniconda
-      - run: *create_conda_env
-      - run: *setup_cdat_info
-      - run: *run_tests
-      - store_artifacts:
-          path: tests_html
-          destination: tests_html
-      - store_artifacts:
-          path: tests_png
-          destination: tests_png
+   macos_cdat_info_py2:
+      macos:
+         xcode: "11.3.0"
+      environment:
+         WORKDIR: /Users/distiller/project/macos_build
+         PKG_NAME: "cdat_info"
+         REPO_NAME: "cdat_info"
+         ENV_NAME: "test_cdat_info"
+         CONDA_PY_VER: "python<3"
+      steps:
+         - checkout
+         - attach_workspace:
+              at: .
+         - run: *setup_run_tests
+         - run: *run_tests
+         - store_artifacts:
+              path: tests_html
+              destination: tests_html
 
-  linux_cdat_info_py3:
-    machine:
-      image: circleci/classic:latest
-    environment:
-      WORKDIR: "workspace/test_linux_cdat_info_py3"
-      OS: "linux-64"
-      PY_VER: "py3"
-      COVERAGE: "-c tests/coverage.json --coverage-from-egg"
-      COVERAGE_PKGS: "coverage coveralls"
-    steps:
-      - checkout
-      - run: *setup_miniconda
-      - run: *create_conda_env
-      - run: *setup_cdat_info
-      - run: *run_tests
-      - run: *run_coveralls
-      - run: *conda_upload
-      - store_artifacts:
-          path: tests_html
-          destination: tests_html
-      - store_artifacts:
-          path: tests_png
-          destination: tests_png
+   macos_cdat_info_py37:
+      macos:
+         xcode: "11.3.0"
+      environment:
+         WORKDIR: /Users/distiller/project/macos_build
+         PKG_NAME: "cdat_info"
+         REPO_NAME: "cdat_info"
+         ENV_NAME: "test_cdat_info"
+         CONDA_PY_VER: "python>=3.7,<3.8"
+      steps:
+         - checkout
+         - attach_workspace:
+              at: .
+         - run: *setup_run_tests
+         - run: *run_tests
+         - store_artifacts:
+              path: tests_html
+              destination: tests_html
+
+   linux_cdat_info_py2:
+      machine:
+         image: circleci/classic:latest
+      environment:
+         WORKDIR: /home/circleci/project/linux_build
+         PKG_NAME: "cdat_info"
+         REPO_NAME: "cdat_info"
+         ENV_NAME: "test_cdat_info"
+         CONDA_PY_VER: "python<3"
+      steps:
+         - checkout
+         - attach_workspace:
+              at: .
+         - run: *setup_run_tests
+         - run: *run_tests
+         - store_artifacts:
+              path: tests_html
+              destination: tests_html
+
+   linux_cdat_info_py37:
+      machine:
+         image: circleci/classic:latest
+      environment:
+         WORKDIR: /home/circleci/project/linux_build
+         PKG_NAME: "cdat_info"
+         REPO_NAME: "cdat_info"
+         ENV_NAME: "test_cdat_info"
+         CONDA_PY_VER: "python>=3.7,<3.8"
+         COVERAGE: "-c tests/coverage.json --coverage-from-egg"
+         COVERAGE_PKGS: "coverage coveralls"
+      steps:
+         - checkout
+         - attach_workspace:
+              at: .
+         - run: *setup_run_tests
+         - run: *run_tests
+         - run: *run_coveralls
+         - store_artifacts:
+              path: tests_html
+              destination: tests_html
+
+   upload:
+      machine:
+         image: circleci/classic:latest
+      environment:
+         WORKDIR: /home/circleci/project/linux_build
+         PKG_NAME: "cdat_info"
+         VERSION: "8.2"
+         USER: "cdat"
+         LABEL: "nightly"
+      steps:
+         - attach_workspace:
+              at: .
+         - run: pwd
+         - run: *conda_upload
 
 workflows:
-  version: 2
-  cdat_info:
-    jobs:
-      - macos_cdat_info_py2
-      - macos_cdat_info_py3
-      - linux_cdat_info_py2
-      - linux_cdat_info_py3
+   version: 2
+   cdat_info:
+      jobs:
+         - macos_setup
+         - linux_setup
+         - macos_cdat_info_py2:
+              requires:
+                 - macos_setup
+         - macos_cdat_info_py37:
+              requires:
+                 - macos_setup
+         - linux_cdat_info_py2:
+              requires:
+                 - linux_setup
+         - linux_cdat_info_py37:
+              requires:
+                 - linux_setup
+         - upload:
+              requires:
+                 - macos_cdat_info_py2
+                 - macos_cdat_info_py37
+                 - linux_cdat_info_py2
+                 - linux_cdat_info_py37