Measure test coverage and report to codecov.io, take 2

.codecov.yml

@@ -0,0 +1,2 @@
+# do not comment with code coverage metrics in github pull requests
+comment: false

.travis.yml

@@ -2,21 +2,18 @@ language: cpp
 
 sudo: false
 
-compiler: clang
-
 addons:
   apt:
     sources:
       - ubuntu-toolchain-r-test
-      - llvm-toolchain-trusty-5.0
     packages:
       - tcsh
       - pkg-config
       - netcdf-bin
       - libnetcdf-dev
       - gfortran
-      - clang-5.0
       - wget
+      - curl
       #- lftp
 
 install:
@@ -28,9 +25,8 @@ install:
   #- "lftp ftp://anonymous:travis@travis-ci.org@ftp.cgd.ucar.edu 
   #-e 'mirror /archive/Model-Data/CICE/ ~/ICEPACK_INPUTDATA; quit'"
 
-
 script:
-  - "./icepack.setup --suite travis_suite --testid travisCItest 
+  - "./icepack.setup --suite travis_suite --testid travisCItest
     --mach travisCI --env gnu &&
     cd travis_suite.travisCItest &&
     ./results.csh"

README.md

@@ -1,5 +1,6 @@
 [![Build Status](https://travis-ci.org/CICE-Consortium/Icepack.svg?branch=master)](https://travis-ci.org/CICE-Consortium/Icepack)
 [![Documentation Status](https://readthedocs.org/projects/cice-consortium-icepack/badge/?version=master)](http://cice-consortium-icepack.readthedocs.io/en/master/?badge=master)
+[![codecov.io](http://codecov.io/github/CICE-Consortium/Icepack/coverage.svg?branch=master)](http://codecov.io/github/CICE-Consortium/Icepack?branch=master)
 
 ## Overview
 This repository contains files for Icepack, the column physics of the sea ice model CICE. Icepack is maintained by the CICE Consortium.  For testing purposes and guidance for including Icepack in other sea ice host models, this repository also includes a driver and basic test suite.

configuration/scripts/machines/Macros.travisCI_gnu

@@ -1,8 +1,8 @@
 #==============================================================================
-# Makefile macros for Travis-CI - gfortran and llvm (clang) compilers
+# Makefile macros for Travis-CI - gfortran and gcc compilers
 #==============================================================================
 
-CPP        := clang-cpp
+CPP        := cpp
 CPPDEFS    := -DFORTRANUNDERSCORE -DNO_R16 -DHAVE_F2008_CONTIGUOUS -DLINUX -DCPRINTEL ${ICE_CPPDEFS}
 CFLAGS     := -c -O2 -fp-model precise   -xHost
 
@@ -30,7 +30,7 @@ ifeq ($(ICE_IOTYPE), netcdf)
     INCLDIR := $(INCLDIR) -I$(NETCDF_PATH)/include
     LIB_NETCDF := $(NETCDF_PATH)/lib
     LIB_PNETCDF := 
-    SLIBS   := -L$(LIB_NETCDF) -lnetcdf 
+    SLIBS   := -L$(LIB_NETCDF) -lnetcdf -lnetcdff
 else
     SLIBS   := 
 endif

configuration/scripts/tests/report_codecov.csh

@@ -0,0 +1,26 @@
+#!/bin/csh -f
+if (`where curl` == "") then
+  (>&2 echo "ERROR: Code coverage reporting (--codecov) needs 'curl' to upload results")
+  exit(1)
+endif
+
+# token from https://codecov.io/gh/CICE-Consortium/Icepack/settings
+setenv CODECOV_TOKEN "df12b574-8dce-439d-8d3b-ed7428d7598a"
+
+# The test-coverage files (*.gcno,*.gcda) must reside next to the source code
+# for the coverage reporting to work. However, the coverage files are created
+# for each test. For that reason, this script will copy the coverage files over
+# to the source directory and report the results, one test at a time. Codecov.io
+# is clever enough to report the cumulative results.
+echo "Looping over test cases and uploading test coverage"
+set testdirs=`ls -d ${ICE_MACHINE_WKDIR}/*`
+foreach dir ($testdirs)
+  echo "## Submitting results from ${dir}"
+  cp $dir/compile/*.{gcno,gcda} ${ICE_SANDBOX}/columnphysics/
+  if ( $status == 0 ) then
+      echo "Uploading coverage results to codecov.io"
+      bash -c "bash <(curl -s https://codecov.io/bash)"
+  else
+      echo "No coverage files found for this test"
+  endif
+end

icepack.setup

@@ -23,6 +23,7 @@ set baseCom = $spval  # Baseline compare
 set baseGen = $spval  # Baseline generate
 set bfbcomp = $spval  # BFB compare
 set report  = 0       # test results reporting
+set codecov = 0       # code coverage measurement and reporting
 #set versno = `grep "release =" doc/source/conf.py | cut -d \' -f 2`
 set versno = `git log | grep 'v[0-9]\.[0-9]\.[0-9]\.[a-z][0-9]*' | head -1 | sed 's/.*\(v[0-9]\.[0-9]\.[0-9]\.[a-z][0-9]*\).*/\1/' `
 
@@ -73,7 +74,7 @@ SYNOPSIS
 
     --suite SUITE -m MACH --testid ID 
         [-e ENV1,ENV2][--acct ACCT][--bdir DIR][--bgen DIR]
-        [--bcmp DIR][--report]
+        [--bcmp DIR][--report][--codecov]
 
 DESCRIPTION
     --help, -h : help
@@ -96,6 +97,7 @@ DESCRIPTION
     --testid   : test ID, user-defined id for testing (REQUIRED with --test or --suite)
     --diff     : generate comparison against another case
     --report   : automatically post results when tests are complete
+    --codecov  : generate and report test coverage metrics when tests are complete
 
 EXAMPLES
     icepack.setup -c caseB -m gordon -e cray -s diag1,debug
@@ -172,6 +174,10 @@ while (1)
     set report = 1
     shift argv
 
+  else if ("$option" == "--codecov") then
+    set codecov = 1
+    shift argv
+
 # arguments with settings
   else
     shift argv
@@ -252,6 +258,11 @@ if ($report == 1 && $testsuite == $spval) then
   exit -1
 endif
 
+if ($codecov == 1 && $testsuite == $spval) then
+  echo "${0}: ERROR in arguments, must use --suite with --codecov"
+  exit -1
+endif
+
 if ($testsuite == $spval) then
   if ("$compilers" =~ "*,*") then
     echo "${0}: ERROR in arguments, cannot set multiple compilers without --suite"
@@ -302,6 +313,10 @@ if ( $testsuite != $spval ) then
     cp -f ${ICE_SCRIPTS}/tests/poll_queue.csh ./${tsdir}
   endif
 
+  if ($codecov == 1) then
+    cp -f ${ICE_SCRIPTS}/tests/report_codecov.csh ./${tsdir}
+  endif
+
 cat >! ./${tsdir}/suite.run << EOF0
 #!/bin/csh -f
 EOF0
@@ -443,6 +458,14 @@ foreach file (env.${machcomp} Macros.${machcomp})
   cp -f -p ${ICE_SCRIPTS}/machines/$file ${casedir}
 end
 
+# append coverage-related flags to Macros if `codecov` is true
+if ($codecov == 1) then
+  printf "FFLAGS += -O0 -fprofile-arcs -ftest-coverage -g\
+CFLAGS += -O0 -coverage -g\
+LDFLAGS += -ftest-coverage -fprofile-arcs -g\n" >> \
+    ${casedir}/Macros.${machcomp}
+endif
+
 # from basic script dir to casescr
 foreach file (parse_namelist.sh parse_settings.sh parse_namelist_from_settings.sh icepack_decomp.csh icepack.run.setup.csh icepack.test.setup.csh)
   if !(-e ${ICE_SCRIPTS}/$file) then
@@ -720,6 +743,10 @@ EOF
     ./results.csh
     ./report_results.csh
   endif
+  if ($codecov == 1) then
+    printf "\nMeasuring test coverage and uploading results to codecov.io\n"
+    ./report_codecov.csh
+  endif
   cd ${ICE_SANDBOX}
 
 endif