add python codecov (#2976)

* include pytest-cov plugin

.coveragerc

@@ -0,0 +1,35 @@
+# https://coverage.readthedocs.io/en/latest/config.html
+
+[run]
+source = src/python/turicreate/
+branch  = True
+parallel = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if False:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+omit =
+    */__init__.py
+    src/python/turicreate/test/*
+    src/python/turicreate/meta/*
+    src/python/turicreate/_cython/*
+    src/python/turicreate/_deps/*_mock.py
+    src/python/turicreate/_connect/*

scripts/.coveragerc-template

@@ -0,0 +1,35 @@
+# https://coverage.readthedocs.io/en/latest/config.html
+
+[run]
+source = {{ source }}
+branch  = True
+parallel = True
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if False:
+    if __name__ == .__main__.:
+
+ignore_errors = True
+
+omit =
+    */__init__.py
+    {{ source }}/test/*
+    {{ source }}/meta/*
+    {{ source }}/_cython/*
+    {{ source }}/_deps/*_mock.py
+    {{ source }}/_connect/*

scripts/requirements.txt

@@ -27,4 +27,5 @@ hypothesis>=4.50,<5.0
 wheel>=0.29.0
 six>=1.12.0
 requests>=2.9.1
-pre-commit>=1.21.0
+pytest-cov>=2.7.1
+pre-commit>=1.21.0

scripts/run_python_test.sh

@@ -18,9 +18,9 @@ WORKSPACE=${SCRIPT_DIR}/..
 
 unset PYTHONPATH
 
-cd $WORKSPACE
+cd "$WORKSPACE"
 scripts/install_python_toolchain.sh
-source scripts/python_env.sh $BUILD_TYPE
+source scripts/python_env.sh "$BUILD_TYPE"
 
 # Make python unit tests print unity server log on exception
 export TURI_VERBOSE=1
@@ -33,26 +33,29 @@ push_ld_library_path() {
 pop_ld_library_path() {
   export LD_LIBRARY_PATH=$OLD_LIBRARY_PATH
 }
+
 push_ld_library_path
-cd $TURI_BUILD_ROOT/src/python
+cd "$TURI_BUILD_ROOT"/src/python
 make -j4
 pop_ld_library_path
 
+# back to WORKSPACE to use default .coveragerc
+cd "$WORKSPACE"
 find . -name "*.xml" -delete
 if ! type "parallel" 2> /dev/null; then
         cmd=""
         if [[ $OSTYPE != msys ]]; then
           cmd=${PYTHON_EXECUTABLE}
         fi
-        cmd="${cmd} ${PYTEST_EXECUTABLE} -v $PYTHONPATH/turicreate/test --junit-xml=alltests.pytest.xml"
-        echo $cmd
+        cmd="${cmd} ${PYTEST_EXECUTABLE} --cov -v $PYTHONPATH/turicreate/test \
+          --junit-xml=alltests.pytest.xml"
         $cmd
 else
         cmd=""
         if [[ $OSTYPE != msys ]]; then
           cmd=${PYTHON_EXECUTABLE}
         fi
-        cmd="${cmd} ${PYTEST_EXECUTABLE} -v -s --junit-xml={}.pytest.xml {}"
+        cmd="${cmd} ${PYTEST_EXECUTABLE} --cov -v -s --junit-xml={}.pytest.xml {}"
         echo "Tests are running in parallel. Output is buffered until job is done..."
-        find turicreate/test -name "*.py" | parallel --group -P 4 $cmd
+        find turicreate/test -name "*.py" | parallel --group -P 4 "$cmd"
 fi

scripts/test_wheel.sh

@@ -7,8 +7,8 @@ SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 WORKSPACE=${SCRIPT_DIR}/..
 
 # The build image version that will be used for testing
-TC_BUILD_IMAGE_1404=$(bash $WORKSPACE/scripts/get_docker_image.sh --ubuntu=14.04)
-TC_BUILD_IMAGE_1804=$(bash $WORKSPACE/scripts/get_docker_image.sh --ubuntu=18.04)
+TC_BUILD_IMAGE_1404=$(bash "$WORKSPACE"/scripts/get_docker_image.sh --ubuntu=14.04)
+TC_BUILD_IMAGE_1804=$(bash "$WORKSPACE"/scripts/get_docker_image.sh --ubuntu=18.04)
 
 unknown_option() {
   echo "Unknown option $1. Exiting."
@@ -40,34 +40,37 @@ while [ $# -gt 0 ]
     --docker-python3.6)     USE_DOCKER=1;DOCKER_PYTHON=3.6;;
     --docker-python3.7)     USE_DOCKER=1;DOCKER_PYTHON=3.7;;
     --help)                 print_help ;;
-    *) unknown_option $1 ;;
+    *) unknown_option "$1" ;;
   esac
   shift
 done
 
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 WORKSPACE=${SCRIPT_DIR}/..
-cd ${WORKSPACE}
+
+# PWD is WORKSPACE
+
+cd "${WORKSPACE}"
 
 # If we are going to run in Docker,
 # send this command into Docker and bail out of here when done.
 if [[ -n "${USE_DOCKER}" ]]; then
   # create the build and test images
   # (this should ideally be a no-op if the image exists and is current)
-  $WORKSPACE/scripts/create_docker_images.sh
+  "$WORKSPACE"/scripts/create_docker_images.sh
 
   # Run the tests inside Docker
   if [[ "${DOCKER_PYTHON}" == "2.7" ]] || [[ "${DOCKER_PYTHON}" == "3.5" ]]; then
     docker run --rm -m=8g \
-      --mount type=bind,source=$WORKSPACE,target=/build,consistency=delegated \
+      --mount type=bind,source="$WORKSPACE",target=/build,consistency=delegated \
       -e "VIRTUALENV=virtualenv --python=python${DOCKER_PYTHON}" \
-      ${TC_BUILD_IMAGE_1404} \
+      "${TC_BUILD_IMAGE_1404}" \
       /build/scripts/test_wheel.sh
   elif [[ "${DOCKER_PYTHON}" == "3.6" ]] || [[ "${DOCKER_PYTHON}" == "3.7" ]]; then
     docker run --rm -m=8g \
-      --mount type=bind,source=$WORKSPACE,target=/build,consistency=delegated \
+      --mount type=bind,source="$WORKSPACE",target=/build,consistency=delegated \
       -e "VIRTUALENV=virtualenv --python=python${DOCKER_PYTHON}" \
-      ${TC_BUILD_IMAGE_1804} \
+      "${TC_BUILD_IMAGE_1804}" \
       /build/scripts/test_wheel.sh
   else
     echo "Invalid docker python version detected: ${DOCKER_PYTHON}"
@@ -77,18 +80,29 @@ if [[ -n "${USE_DOCKER}" ]]; then
   exit 0
 fi
 
-(test -d deps/env) || ./scripts/install_python_toolchain.sh
+test -d deps/env || ./scripts/install_python_toolchain.sh
 source deps/env/bin/activate
 pip install target/turicreate-*.whl
 
-PYTHON="$PWD/deps/env/bin/python"
+PYTHON="$WORKSPACE/deps/env/bin/python"
 PYTHON_MAJOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.major)')
 PYTHON_MINOR_VERSION=$(${PYTHON} -c 'import sys; print(sys.version_info.minor)')
 PYTHON_VERSION="python${PYTHON_MAJOR_VERSION}.${PYTHON_MINOR_VERSION}"
-cp -a src/python/turicreate/test deps/env/lib/${PYTHON_VERSION}/site-packages/turicreate/
-cd deps/env/lib/${PYTHON_VERSION}/site-packages/turicreate/test
+
+cp -a src/python/turicreate/test deps/env/lib/"${PYTHON_VERSION}"/site-packages/turicreate/
+
+TEST_DIR="$WORKSPACE"/deps/env/lib/"${PYTHON_VERSION}"/site-packages/turicreate/test
+SOURCE_DIR=$(dirname "$TEST_DIR")
+
+$PYTHON -c "import sys; lines=sys.stdin.read(); \
+  print(lines.replace('{{ source }}', \"$SOURCE_DIR\"))" \
+  < scripts/.coveragerc-template \
+  > "${TEST_DIR}"/.coveragerc
+
+cd "$TEST_DIR"
 
 # run tests
-${PYTHON} -m pytest -v --durations=100 --junit-xml=../../../../../../../pytest.xml
+${PYTHON} -m pytest --cov -v --durations=100 \
+  --junit-xml="$WORKSPACE"/pytest.xml
 
 date