refactor CI logic

Jenkinsfile

@@ -1,77 +1,20 @@
-def prepare_clean_env(env_name) {
-  sh """#!/bin/bash
-  printenv
-  git clean -ff -d -x --exclude='tests/externaldata/*' --exclude='tests/data/*' --exclude=conda
-  conda env update --prune -p conda/${env_name} -f env/${env_name}.yml
-  conda activate ./conda/${env_name}
-  conda list
-  make clean
-  """
-}
-
-def install_dep(env_name) {
-  sh """#!/bin/bash
-  conda activate ./conda/${env_name}
-  python setup.py install --force
-  python -m spacy download en
-  python -m nltk.downloader all
-  """
-}
-
-def run_test(env_name, folders, nproc, serial, capture_opt) {
-  sh script:"""#!/bin/bash
-  conda activate ./conda/${env_name}
-  printenv
-  py.test -v ${capture_opt} -n ${nproc} -m "${serial}" --durations=50 --cov=./ ${folders}
-  """
-}
-
-def report_cov(name, flag) {
-  sh """#!/bin/bash
-  bash ./codecov.sh -c -F ${flag} -n ${name}
-  """
-}
-
-def upload_doc(bucket, path) {
-  sh """#!/bin/bash
-  echo "Uploading doc to s3://${bucket}/${path}/"
-  aws s3 sync --delete docs/_build/html/ s3://${bucket}/${path}/ --acl public-read
-  echo "Uploaded doc to http://${bucket}.s3-accelerate.dualstack.amazonaws.com/${path}/index.html"
-  """
-}
-
 stage("Sanity Check") {
   node {
     ws('workspace/gluon-nlp-lint') {
       checkout scm
-      prepare_clean_env('pylint')
-      sh """#!/bin/bash
-      conda activate ./conda/pylint
-      printenv
-      make lint
-      """
+      sh('ci/step_sanity_check.sh')
     }
   }
 }
 
 stage("Unit Test") {
+  capture_flag = env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no'
   parallel 'Python 2': {
     node {
       withCredentials([string(credentialsId: 'GluonNLPCodeCov', variable: 'CODECOV_TOKEN')]) {
         ws('workspace/gluon-nlp-py2') {
           checkout scm
-          prepare_clean_env('py2')
-          install_dep('py2')
-          run_test('py2', 'tests/unittest', '4', 'not serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          run_test('py2', 'tests/unittest', '0', 'serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          report_cov(env.BRANCH_NAME+'-py2', 'unittests')
-          run_test('py2', 'scripts', '4', 'not serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          run_test('py2', 'scripts', '0', 'serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          report_cov(env.BRANCH_NAME+'-py2', 'integration')
+          sh("ci/step_unit_test.sh py2 ${env.BRANCH_NAME} ${capture_flag}")
         }
       }
     }
@@ -81,18 +24,7 @@ stage("Unit Test") {
       withCredentials([string(credentialsId: 'GluonNLPCodeCov', variable: 'CODECOV_TOKEN')]) {
         ws('workspace/gluon-nlp-py3') {
           checkout scm
-          prepare_clean_env('py3')
-          install_dep('py3')
-          run_test('py3', 'tests/unittest', '4', 'not serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          run_test('py3', 'tests/unittest', '0', 'serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          report_cov(env.BRANCH_NAME+'-py3', 'unittests')
-          run_test('py3', 'scripts', '4', 'not serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          run_test('py3', 'scripts', '0', 'serial',
-           env.BRANCH_NAME.startsWith('PR-')?'':'--capture=no')
-          report_cov(env.BRANCH_NAME+'-py3', 'integration')
+          sh("ci/step_unit_test.sh py3 ${env.BRANCH_NAME} ${capture_flag}")
         }
       }
     }
@@ -102,20 +34,16 @@ stage("Unit Test") {
       ws('workspace/gluon-nlp-docs') {
         checkout scm
         retry(3) {
-          prepare_clean_env('doc')
-          install_dep('doc')
-          sh """#!/bin/bash
-          conda activate ./conda/doc
-          export LD_LIBRARY_PATH=/usr/local/cuda/lib64
-          printenv
-          make docs
-          """
+          sh("ci/step_documentation.sh")
         }
         if (env.BRANCH_NAME.startsWith("PR-")) {
-          upload_doc('gluon-nlp-staging', env.BRANCH_NAME+'/'+env.BUILD_NUMBER)
+          bucket = 'gluon-nlp-staging'
+          path = env.BRANCH_NAME+'/'+env.BUILD_NUMBER
         } else {
-          upload_doc('gluon-nlp', env.BRANCH_NAME)
+          bucket = 'gluon-nlp'
+          path = env.BRANCH_NAME
         }
+        sh("ci/upload_doc.sh ${bucket} ${path}")
       }
     }
   }

Makefile

@@ -54,7 +54,9 @@ docs: release
 	done;
 
 clean:
-	git clean -f -d -x --exclude="$(ROOTDIR)/tests/externaldata/*" --exclude=conda
+	git clean -ff -d -x --exclude="$(ROOTDIR)tests/externaldata/*" --exclude="$(ROOTDIR)/tests/data/*" --exclude=conda
+
+clean_doc:
 	make -C docs clean
 
 compile_notebooks:

ci/install_dep.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+python setup.py install --force
+python -m spacy download en
+python -m nltk.downloader all

ci/prepare_clean_env.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+env_name=$1
+make clean
+conda env update --prune -p conda/${env_name} -f env/${env_name}.yml
+conda activate ./conda/${env_name}
+conda list
+printenv

ci/step_documentation.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+export LD_LIBRARY_PATH=/usr/local/cuda/lib64
+source ci/prepare_clean_env.sh doc
+ci/install_dep.sh
+make clean_doc
+make docs
+set +ex

ci/step_sanity_check.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+source ci/prepare_clean_env.sh pylint
+make lint

ci/step_unit_test.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+lang=$1
+branch=$2
+capture_flag=$3
+set -ex
+source ci/prepare_clean_env.sh ${lang}
+ci/install_dep.sh
+py.test -v ${capture_flag} -n 4 -m "not serial" --durations=50 --cov=gluonnlp tests/unittest
+py.test -v ${capture_flag} -n 0 -m "serial" --durations=50 --cov=gluonnlp tests/unittest
+ci/codecov.sh -c -F ${branch}-${lang} -n unittests
+py.test -v ${capture_flag} -n 4 -m "not serial" --durations=50 --cov=gluonnlp --cov=scripts scripts
+py.test -v ${capture_flag} -n 0 -m "serial" --durations=50 --cov=gluonnlp --cov=scripts scripts
+ci/codecov.sh -c -F ${branch}-${lang} -n integration
+set +ex

ci/upload_doc.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+bucket=$1
+path=$2
+echo "Uploading doc to s3://${bucket}/${path}/"
+aws s3 sync --delete docs/_build/html/ s3://${bucket}/${path}/ --acl public-read
+echo "Uploaded doc to http://${bucket}.s3-accelerate.dualstack.amazonaws.com/${path}/index.html"

env/py2.yml

@@ -9,6 +9,7 @@ dependencies:
   - pytest
   - flaky
   - pytest-cov
+  - mock
   - pytest-xdist
   - pip=9.0.3
   - pip:

env/py3.yml

@@ -9,6 +9,7 @@ dependencies:
   - pytest
   - flaky
   - pytest-cov
+  - mock
   - pytest-xdist
   - pip=9.0.3
   - pip: