Merge branch 'deviate/jenkins' into feature/exposures_crs

Makefile

@@ -2,6 +2,9 @@
 # test, coverage and lint
 ###
 
+# Default pytest command
+PYTEST_CMD := pytest
+
 PYTEST_JUNIT_ARGS = --junitxml=tests_xml/tests.xml
 
 PYTEST_COV_ARGS = \
@@ -16,15 +19,15 @@ help:  ## Use one of the following instructions:
 
 .PHONY : lint
 lint : ## Static code analysis with Pylint
-	pylint -ry climada_petals > pylint.log || true
+	pylint -ry climada_petals > pylint.log
 
 .PHONY : unit_test
 unit_test : ## Unit tests execution with coverage and xml reports
-	pytest $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/
+	$(PYTEST_CMD) $(PYTEST_ARGS) --ignore=climada_petals/test climada_petals/
 
 .PHONY : install_test
 install_test : ## Test installation was successful
-	pytest $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \
+	$(PYTEST_CMD) $(PYTEST_JUNIT_ARGS) --pyargs climada.engine.test.test_cost_benefit \
 	climada.engine.test.test_impact
 
 .PHONY : data_test
@@ -37,11 +40,11 @@ notebook_test : ## Test notebooks in doc/tutorial
 
 .PHONY : integ_test
 integ_test : ## Integration tests execution with xml reports
-	pytest $(PYTEST_ARGS) climada_petals/test/
+	$(PYTEST_CMD) $(PYTEST_ARGS) climada_petals/test/
 
 .PHONY : test
 test : ## Unit and integration tests execution with coverage and xml reports
-	pytest $(PYTEST_ARGS) climada_petals/
+	$(PYTEST_CMD) $(PYTEST_ARGS) climada_petals/
 
 .PHONY : ci-clean
 ci-clean :

script/jenkins/branches/Jenkinsfile

@@ -1,36 +1,31 @@
 pipeline {
   agent any
 
+  environment {
+    FULL_PATH_BRANCH = "${sh(script:'git name-rev --name-only HEAD', returnStdout: true)}"
+    GIT_BRANCH = FULL_PATH_BRANCH.substring(FULL_PATH_BRANCH.lastIndexOf('/') + 1, FULL_PATH_BRANCH.length())
+  }
+
   stages {
+    stage('setup-environment') {
+      steps {
+        sh "bash script/jenkins/branches/run_unittest.sh ${env.GIT_BRANCH}"
+      }
+    }
     stage('ci') {
       parallel {
 
         stage('lint') {
           steps {
-            sh '''#!/bin/bash
-            export PATH=$PATH:$CONDAPATH
-            source activate petals_env
-            rm -f pylint.log
-            pylint -ry climada_petals | tee pylint.log'''
-
+            sh "bash script/jenkins/branches/make_in_venv.sh ${env.GIT_BRANCH} lint"
             discoverGitReferenceBuild referenceJob: 'petals_branches/develop'
             recordIssues tools: [pyLint(pattern: 'pylint.log')]
           }
         }
 
         stage('unit_test') {
           steps {
-            sh '''#!/bin/bash
-            export PATH=$PATH:$CONDAPATH
-            source activate
-
-            # first install xemsf, then activate environment, so .../envs/.../etc/conda/activate.d/esmpy.sh is executed
-            # otherwise the environment varibalbe ESMFMKFILE is not initialized
-            mamba install -n petals_env -c conda-forge -y xesmf ruamel.yaml cdsapi
-            conda activate petals_env
-            rm -rf tests_xml/
-            rm -rf coverage/
-            make unit_test'''
+            sh "bash script/jenkins/branches/make_in_venv.sh ${env.GIT_BRANCH} unit_test PYTEST_COMMAND=\"python -m pytest\""
           }
         }
 

script/jenkins/branches/make_in_venv.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+export PATH=$PATH:$CONDAPATH
+
+source activate petals_env
+rm -rf tests_xml/
+rm -rf coverage/
+
+BRANCH=$1
+shift
+CORENV=~/jobs/petals_branches/core_env
+if [ -f $CORENV/$BRANCH ]; then
+    source tvenv/bin/activate
+fi
+
+make $@
+
+if [ -f $CORENV/$BRANCH ]; then
+    deactivate
+fi
+
+conda deactivate

script/jenkins/branches/setup_venv.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+export PATH=$PATH:$CONDAPATH
+
+BRANCH=$1
+CORENV=~/jobs/petals_branches/core_env
+
+echo $CORENV/$BRANCH
+
+if [ -f $CORENV/$BRANCH ]; then
+    echo file exists
+    cat $CORENV/$BRANCH
+
+    source activate petals_env
+
+    python -m venv --system-site-packages tvenv
+    source tvenv/bin/activate
+
+    pip install -e `cat $CORENV/$BRANCH`
+
+    deactivate
+    conda deactivate
+fi