Add makefile

.circleci/config.yml

@@ -12,23 +12,18 @@ aliases:
   - &conda_rerender
     name: conda_rerender
     command: |
-       git clone https://github.com/CDAT/conda-recipes.git $WORKDIR/conda-recipes 
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       conda config --set anaconda_upload no
-       BUILD_SCRIPT="$WORKDIR/conda-recipes/build_tools/conda_build.py"
-       python $BUILD_SCRIPT -w $WORKDIR -l $LAST_STABLE -B 0 -p $PKG_NAME -b $CIRCLE_BRANCH --do_rerender
+
+       make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH
 
   - &conda_build
     name: conda_build
     command: |
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       conda config --add channels conda-forge --force
-       conda config --add channels cdat/label/nightly --force
-       conda config --set channel_priority strict
-       BUILD_SCRIPT="$WORKDIR/conda-recipes/build_tools/conda_build.py"
-       python $BUILD_SCRIPT -w $WORKDIR -p $PKG_NAME --build_version $BUILD_VARIANT_VER --do_build
+
+       make conda-build conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR artifcat_dir=$PWD/artifacts copy_conda_package=1
 
   - &setup_run_tests
     name: setup_run_tests
@@ -41,40 +36,31 @@ aliases:
     command: |
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       if [[ `uname` == "Linux" ]]; then
-          MESA="mesalib=18.3.1"
-       else
-          MESA="mesalib=17.3.9"
-       fi
-       echo "conda create -y -n $ENV_NAME --use-local $CHANNELS \"$CONDA_PY_VER\" $PKG_NAME $MESA $PKGS $DOC_PKGS $COVERAGE_PKGS \"$LIBNETCDF\""
-       conda create -y -n $ENV_NAME --use-local $CHANNELS "$CONDA_PY_VER" $PKG_NAME $MESA $PKGS $DOC_PKGS $COVERAGE_PKGS "$LIBNETCDF"
-       conda activate $ENV_NAME
-       conda config --set always_yes yes
-       conda install -c conda-forge nb_conda nb_conda_kernels
-       conda list
+
+       make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs="$CONDA_PY_VER $LIBNETCDF"
+
+       make conda-dump-env conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME artifcat_dir=$PWD/artifacts conda_env_name=$CIRCLE_JOB
 
   - &get_testdata
     name: get_testdata
     command: |
-      git clone git://github.com/cdat/uvcdat-testdata
+      make get-testdata conda=$WORKDIR/miniconda/bin/conda 
 
   - &run_tests
     name: run_tests
     command: |
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
-       conda activate $ENV_NAME
-       python run_tests.py -n 4 -H -v2 --timeout=100000 --checkout-baseline --no-vtk-ui $COVERAGE
-       RESULT=$?
-       conda deactivate
-       exit $RESULT
+       conda activate base
+
+       make run-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME
 
   - &conda_env_export
     name: conda_env_expprt
     command: |
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
-       conda activate $ENV_NAME
-       conda env export > $WORKDIR/env.yml
-       cat $WORKDIR/env.yml
+       conda activate base
+
+       make conda-list conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME
 
   - &conda_upload
     name: conda_upload
@@ -84,27 +70,24 @@ aliases:
        fi
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
        conda activate base
-       UPLOAD_OPTIONS="-t $CONDA_UPLOAD_TOKEN upload -u $USER -l $LABEL"
-       anaconda $UPLOAD_OPTIONS $WORKDIR/miniconda/conda-bld/noarch/$PKG_NAME-$VERSION.`date +%Y*`0.tar.bz2 --force
+
+       make conda-upload conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME conda_upload_token=$CONDA_UPLOAD_TOKEN user=$USER label=$LABEL
 
   - &run_coveralls
     name: run_coveralls
     command: |
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
-       conda activate $ENV_NAME
-       coveralls
-       conda deactivate
+       conda activate base
+
+       make run-coveralls conda=$WORKDIR/miniconda/bin/conda 
 
   - &make_doc_test
     name: make_doc_test
     command: |
        source $WORKDIR/miniconda/etc/profile.d/conda.sh
-       conda activate $ENV_NAME
-       cd docs
-       make doctest
-       DOC_RESULT=$?
-       echo "*** $PY_VER doc test result: "${DOC_RESULT}
-       exit $DOC_RESULT
+       conda activate base
+
+       make run-doc-test conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME
     no_output_timeout: 20m
 
 
@@ -123,6 +106,8 @@ jobs:
          - run: *setup_miniconda
          - run: *conda_rerender
          - run: *conda_build
+         - store_artifacts:
+              path: artifacts
          - persist_to_workspace:
               root: .
               paths: 
@@ -142,6 +127,8 @@ jobs:
          - run: *setup_miniconda
          - run: *conda_rerender
          - run: *conda_build
+         - store_artifacts:
+              path: artifacts
          - persist_to_workspace:
               root: .
               paths:
@@ -164,6 +151,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *get_testdata
          - run: *run_tests
          - store_artifacts:
@@ -187,6 +176,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *get_testdata
          - run: *run_tests
          - store_artifacts:
@@ -210,6 +201,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *get_testdata
          - run: *run_tests
          - store_artifacts:
@@ -233,6 +226,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -259,6 +254,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -285,6 +282,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -312,6 +311,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -336,6 +337,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -360,6 +363,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -384,6 +389,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -408,6 +415,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -433,6 +442,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -460,6 +471,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -484,6 +497,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -508,6 +523,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -532,6 +549,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -556,6 +575,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -581,6 +602,8 @@ jobs:
          - attach_workspace:
               at: .
          - run: *setup_run_tests
+         - store_artifacts:
+              path: artifacts
          - run: *conda_env_export
          - run: *get_testdata
          - run: *run_tests
@@ -621,6 +644,8 @@ jobs:
          - run: *conda_upload
 
 workflows:
+   # Might be work looking into matrix to simplify, if we can use version 2.1.
+   # https://circleci.com/docs/2.0/configuration-reference/#matrix-requires-version-21
    version: 2
    vcs:
       jobs:

Makefile

@@ -0,0 +1,93 @@
+.PHONY: conda-info conda-list setup-build setup-tests conda-rerender \
+	conda-build conda-upload conda-dump-env conda-cp-output get_testdata \
+	run-tests run-doc-tests run-coveralls
+
+SHELL = /bin/bash
+
+os = $(shell uname)
+pkg_name = vcs
+build_script = conda-recipes/build_tools/conda_build.py
+
+test_pkgs = udunits2 testsrunner matplotlib image-compare nbformat ipywidgets \
+						nb_conda nb_conda_kernels coverage coveralls
+docs_pkgs = sphinxcontrib-websupport nbsphinx jupyter_client jupyterlab vcsaddons 
+ifeq ($(os),Linux)
+pkgs = "mesalib=18.3.1"
+else
+pkgs = "mesalib=17.3.9"
+endif
+
+conda_env ?= base
+workdir ?= $(PWD)/workspace
+last_stable ?= 8.2
+branch ?= $(shell git rev-parse --abbrev-ref HEAD)
+extra_channels ?= cdat/label/nightly conda-forge
+conda ?= $(or $(CONDA_EXE),$(shell find /opt/*conda*/bin $(HOME)/*conda* -type f -iname conda))
+artifact_dir ?= $(PWD)/artifacts
+conda_env_filename ?= spec-file
+# TODO change back to master
+conda_recipes_branch ?= build_tool_update
+
+conda_base = $(patsubst %/bin/conda,%,$(conda))
+conda_activate = $(conda_base)/bin/activate
+
+ifneq ($(copy_conda_package),)
+conda_build_extra = --copy_conda_package $(artifact_dir)/
+endif
+
+conda-info:
+	source $(conda_activate) $(conda_env); conda info
+
+conda-list:
+	source $(conda_activate) $(conda_env); conda list
+
+setup-build:
+ifeq ($(wildcard $(workdir)/conda-recipes),)
+	git clone -b $(conda_recipes_branch) https://github.com/CDAT/conda-recipes $(workdir)/conda-recipes
+else
+	cd $(workdir)/conda-recipes; git pull
+endif
+
+setup-tests:
+	source $(conda_activate) base; conda create -y -n $(conda_env) --use-local \
+		$(foreach x,$(extra_channels),-c $(x)) $(pkg_name) $(foreach x,$(test_pkgs),"$(x)") \
+		$(foreach x,$(docs_pkgs),"$(x)") $(foreach x,$(pkgs),"$(x)") $(foreach x,$(extra_pkgs),"$(x)")
+
+conda-rerender: setup-build 
+	python $(workdir)/$(build_script) -w $(workdir) -l $(last_stable) -B 0 -p $(pkg_name) \
+		-b $(branch) --do_rerender --conda_env $(conda_env) --ignore_conda_missmatch \
+		--conda_activate $(conda_activate)
+
+conda-build:
+	mkdir -p $(artifact_dir)
+
+	python $(workdir)/$(build_script) -w $(workdir) -p $(pkg_name) --build_version noarch \
+		--do_build --conda_env $(conda_env) --extra_channels $(extra_channels) \
+		--conda_activate $(conda_activate) $(conda_build_extra)
+
+conda-upload:
+	source $(conda_activate) $(conda_env); \
+		output=$$(conda build --output $(workdir)/vcs); \
+		anaconda -t $(conda_upload_token) upload -u $(user) -l $(label) $${output} --force
+
+conda-dump-env:
+	mkdir -p $(artifact_dir)
+
+	source $(conda_activate) $(conda_env); conda list --explicit > $(artifact_dir)/$(conda_env_filename).txt
+
+get-testdata:
+ifeq ($(wildcard uvcdat-testdata),)
+	git clone https://github.com/CDAT/uvcdat-testdata
+else
+	cd uvcdat-testdata; git pull
+endif
+
+run-tests:
+	source $(conda_activate) $(conda_env); python run_tests.py -n 4 -H -v2 --timeout=100000 \
+		--checkout-baseline --no-vtk-ui
+
+run-doc-test:
+	source $(conda_activate) $(conda_env); cd docs; make doctest;
+
+run-coveralls:
+	source $(conda_activate) $(conda_env); coveralls;