civi-test-run

#!/bin/bash
set -e

function getPhpunitVer() {
  CIVIVER=$(getCiviVer)
  if [ $(php -r "echo version_compare('$CIVIVER', '5.39', '>=');") ]; then
    echo "phpunit8 --printer \Civi\Test\TAP"
  elif [ $(php -r "echo version_compare('$CIVIVER', '5.28', '>=');") ]; then
    echo "phpunit7 --printer \Civi\Test\TAP"
  elif [ $(php -r "echo version_compare('$CIVIVER', '5.22', '>=');") ]; then
    echo "phpunit6 --printer \Civi\Test\TAP"
  else
    echo 'phpunit5 --tap'
  fi
}

#################################################
## Helpers
function fatal() {
  echo "Fatal: $@"
  exit 1
}

function getCiviVer() {
  pushd "$CIVI_CORE" >> /dev/null
    if [ -f xml/version.xml ]; then
      ## Works in any git-based build, even if gencode hasn't run yet.
      php -r 'echo simplexml_load_file("xml/version.xml")->version_no;'
    else
      ## works in any tar-based build.
      php -r 'require "civicrm-version.php"; $a = civicrmVersion(); echo $a["version"];'
    fi
  popd >> /dev/null
}

function show_help() {
  PROG=$(basename "$0")
  echo "about: Execute the CiviCRM test suites"
  echo
  echo "usage: $PROG [options] <suites>"
  echo
  echo "options:"
  echo "  -h                  Display help"
  echo "  -b <build-name>     The name of a local site produced by civibuild (required)"
  echo "  -j <junit-dir>      The path to a folder for storing results in JUnit XML (required)"
  echo "  --exclude-group <g> Exclude tests with a particular @group"
  echo "  <suites>            A list of one more of the following:"
  echo "                        - all"
  echo "                        - karma"
  echo "                        - mixin"
  echo "                        - phpunit-api3"
  echo "                        - phpunit-api4"
  echo "                        - phpunit-civi"
  echo "                        - phpunit-crm"
  echo "                        - phpunit-e2e"
  echo "                        - phpunit-drupal"
  echo "                        - phpunit-wordpress"
  echo "                        - phpunit-backdrop"
  echo "                        - phpunit-creditnotes"
  echo "                        - phpunit-flexmailer"
  echo "                        - upgrade"
  echo "                        - upgrade@4.7.30:10"
  echo
  echo "example: Run the KarmaJS and DB-upgrade tests on D7/Civi-Master build"
  echo "  $PROG -b dmaster -j /tmp/junit-output karma upgrade"
  echo
}

#################################################
## Test tasks
##
## Each task is responsible for depositing a JUnit XML file under
## "$JUNITDIR".

function task_phpunit() {
  SUITE="$1"

  if ! $GUARD phpunit-each $EXCLUDE_GROUP "$CIVI_CORE" "$JUNITDIR" $SUITE ; then
    EXITCODES="$EXITCODES phpunit"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""

  if [ $SUITE = 'E2E_AllTests' ]; then
      $GUARD civibuild restore "$BLDNAME"
  fi

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
}

## usage: task_phpunit_core_extension <extension-dir> <junit-file> [<phpunit-group>]
function task_phpunit_core_extension() {

 EXTENSION="$1"
 if [ ! -z "$2" ]; then
   FILENAME="junit-$2.xml"
 else
   FILENAME="junit-$EXTENSION.xml"
 fi
 if [ -z "$3" ]; then
   GROUP=""
 else
   GROUP="--group $3"
 fi

 pushd "$CIVI_CORE/ext/$EXTENSION"
 if ! $GUARD $(getPhpunitVer) --printer='Civi\Test\TAP' --log-junit="$JUNITDIR/$FILENAME" $GROUP; then
   EXITCODES="$EXITCODES $EXTENSION"
 fi
 echo "Found EXITCODES=\"$EXITCODES\""

 $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml

}

function task_mixin() {
  pushd "$CIVI_CORE"
    if [ -f tools/mixin/bin/test-all ]; then
      if ! $GUARD ./tools/mixin/bin/test-all "$JUNITDIR" ; then
        EXITCODES="$EXITCODES mixin"
      fi
      $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
    else
      junit_placeholder
    fi
  popd
}

function test_core_extenions() {
  for DIR in $CIVI_CORE/ext/*/
  do
    if [ "$(basename $DIR)" = "afform" ]; then
      test_afform_extensions $DIR
    elif [ -f "$DIR/phpunit.xml.dist" ]; then
      task_phpunit_core_extension "$(basename $DIR)"
      $GUARD civibuild restore "$BLDNAME"
    fi
  done
}

function test_afform_extensions() {
  DIR="$1"
  task_phpunit_core_extension "$(basename $DIR)/core" "afform-core"
  $GUARD civibuild restore "$BLDNAME"
  cv en afform
  cv en --ignore-missing authx
  task_phpunit_core_extension "$(basename $DIR)/mock" "afform-mock-e2e" e2e
  task_phpunit_core_extension "$(basename $DIR)/mock" "afform-mock-headless" headless
  $GUARD civibuild restore "$BLDNAME"
}

function task_karma() {
  $GUARD pushd "$CIVI_CORE"
    ## Work-around: ensure pradmin user has sa contact
    ADMIN_USER=$(cv ev 'echo $GLOBALS["_CV"]["ADMIN_USER"];')
    cv api -U "$ADMIN_USER" contact.get id=user_contact_id
    if [ -f karma.conf.js ]; then
      if ! $GUARD karma start --browsers PhantomJS --single-run --reporters dots,junit ; then
        EXITCODES="$EXITCODES karma"
      fi
      echo "Found EXITCODES=\"$EXITCODES\""
      $GUARD cp tests/output/karma.xml "$JUNITDIR/"
    fi
  $GUARD popd
}

## Execute the upgrade test suite
## ex: task_upgrade @4.3..4.6.30
function task_upgrade() {
  ## Run the tests -- DB upgrade tests
  if  ! $GUARD civibuild upgrade-test $BLDNAME $@ ; then
    EXITCODES="$EXITCODES upgrade-test"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""
  if [ -d "$PRJDIR/build/$BLDNAME/.civibuild" ]; then
    $GUARD cp "$PRJDIR/build/$BLDNAME/.civibuild/debug/civicrm-upgrade-test.xml" "$JUNITDIR/"
  else
    $GUARD cp "$PRJDIR/app/debug/$BLDNAME/civicrm-upgrade-test.xml" "$JUNITDIR/"
  fi
  $GUARD civibuild restore "$BLDNAME"
}

#################################################
## Cleanup left-overs from previous test-runs
function junit_cleanup() {
  [ -d "$JUNITDIR" ] && $GUARD rm -rf "$JUNITDIR"
  [ ! -d "$JUNITDIR" ] && $GUARD mkdir "$JUNITDIR"
}

#################################################
## Create a stub JUnit XML file. This is useful if we've decided to explicitly *not* run tests.
function junit_placeholder() {
  echo '<?xml version="1.0" encoding="UTF-8"?>' > "$JUNITDIR/none.xml"
  echo '<testsuites>' >> "$JUNITDIR/none.xml"
  echo '</testsuites>' >> "$JUNITDIR/none.xml"
}

function task_phpunit_drupal() {
  $GUARD pushd "$CIVI_CORE/drupal"
  if ! $GUARD $(getPhpunitVer) --log-junit "$JUNITDIR/phpunit_drupal.xml" $EXCLUDE_GROUP ; then
    EXITCODES="$EXITCODES phpunit-drupal"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
  $GUARD popd
}

function task_phpunit_wordpress() {
  $GUARD pushd "$CIVI_CORE/.."
  if ! $GUARD $(getPhpunitVer) --log-junit "$JUNITDIR/phpunit_wordpress.xml" $EXCLUDE_GROUP ; then
    EXITCODES="$EXITCODES phpunit-wordpress"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
  $GUARD popd
}

function task_phpunit_backdrop() {
  $GUARD pushd "$CIVI_CORE/backdrop"
  if ! $GUARD $(getPhpunitVer) --log-junit "$JUNITDIR/phpunit_backdrop.xml" $EXCLUDE_GROUP ; then
    EXITCODES="$EXITCODES phpunit-backdrop"
  fi
  echo "Found EXITCODES=\"$EXITCODES\""

  $GUARD phpunit-xml-cleanup "$JUNITDIR"/*.xml
  $GUARD popd
}

## Determine the absolute path of the directory with the file
## absdirname <file-path>
function absdirname() {
  pushd $(dirname $0) >> /dev/null
    pwd
  popd >> /dev/null
}

BINDIR=$(absdirname "$0")
PRJDIR=$(dirname "$BINDIR")
[ -z "$CIVIBUILD_HOME" ] && TMPDIR="$PRJDIR/app/tmp" || TMPDIR="$CIVIBUILD_HOME/.civibuild/tmp"
[ -z "$CIVIBUILD_HOME" ] && BLDDIR="$PRJDIR/build" || BLDDIR="$CIVIBUILD_HOME"
EXITCODES=
SUITES=
EXCLUDE_GROUP=
GUARD=

[ -f "$PRJDIR/app/civibuild.conf" ] && source "$PRJDIR/app/civibuild.conf"

#################################################
## Parse inputs
while [ -n "$1" ] ; do
  OPTION="$1"
  shift

  case "$OPTION" in
    -b)
      BLDNAME="$1"
      [ -d "$BLDDIR/$BLDNAME/web" ] && CMS_ROOT="$BLDDIR/$BLDNAME/web" || CMS_ROOT="$BLDDIR/$BLDNAME"
      pushd "$CMS_ROOT" >> /dev/null
        CIVI_CORE=$(cv ev 'echo $GLOBALS["civicrm_root"];')
        ## or: cv path -d '[civicrm.root]'
      popd >> /dev/null
      shift
      ;;

    -j)
      JUNITDIR="$1"
      junit_cleanup
      shift
      ;;

    --exclude-group)
      EXCLUDE_GROUP="--exclude-group $1"
      shift
      ;;

    -h|--help|-?)
      show_help
      exit 2
      ;;

    *)
      SUITES=$(echo "$SUITES" "$OPTION" | xargs)
      # note: xargs strips leading/trailing spaces
      ;;
  esac
done

## Validate
if [ -z "$SUITES" -o -z "$BLDNAME" -o -z "$JUNITDIR" ]; then
  echo "Error: Missing required argument"
  echo
  show_help
  exit 2
fi

if [ "$SUITES" = "all" ]; then
  pushd "$CIVI_CORE" >> /dev/null
    SUITES="upgrade"
    if [ -f "karma.conf.js" ]; then
      SUITES="karma $SUITES"
    else
      echo "Skip unavailable suite: karma"
    fi

    if [ -f "tests/phpunit/E2E/AllTests.php" ]; then
      SUITES="$SUITES phpunit-e2e"
    else
      echo "Skip unavailable suite: phpunit-e2e"
    fi

    SUITES="$SUITES phpunit-crm phpunit-api3 phpunit-civi"

    if [ -f "tests/phpunit/api/v4/AllTests.php" ]; then
      SUITES="$SUITES phpunit-api4"
    fi

    if [ -d "ext" ]; then
      SUITES="$SUITES phpunit-core-exts"
    fi

    if [ -f "tools/mixin/bin/test-all" ]; then
      SUITES="$SUITES mixin"
    else
      echo "Skip unavailable suite: mixin"
    fi

  popd >> /dev/null
elif [[ " $SUITES " =~ \ all\  ]]; then
  echo "The \"all\" target should not be mixed with other targets."
  exit 3
fi

## dev/core#1781: Some tests exhibit flakiness due to the timing/pacing/happenstance of execution. TIME_FUNC helps to deterministically simulate some of this variation.
if [ -n "$TIME_FUNC" ]; then
  echo "(Clock simulation) Tests will use inherited option TIME_FUNC=$TIME_FUNC"
else
  TIME_FUNC=$( php -r '$a=$argv; array_shift($a); shuffle($a); echo $a[0];' natural natural frozen linear:500 linear:1250 prng:500 prng:666 prng:1000 prng:1500 )
  echo "(Clock simulation) No TIME_FUNC was specified. Using TIME_FUNC=$TIME_FUNC"
fi
export TIME_FUNC

## Main
for TESTTYPE in $SUITES ; do
  case "$TESTTYPE" in
    karma)  task_karma ;;
    mixin) task_mixin ;;
    upgrade)  CIVIVER=$(getCiviVer) ; task_upgrade 5.13.3-multilingual_af_bg_en* "@4.5.9..$CIVIVER:10" ;;
    upgrade@*)  task_upgrade $(echo $TESTTYPE | sed s'/^upgrade//' ) ;;
    phpunit-e2e)  PHP_APC_CLI=on task_phpunit E2E_AllTests ;;
    phpunit-crm)  task_phpunit CRM_AllTests ;;
    # Handle legacy call for phpunit-api being equivilant to phpunit-api3
    phpunit-api)  task_phpunit api_v3_AllTests ;;
    phpunit-api3) task_phpunit api_v3_AllTests ;;
    phpunit-api4) task_phpunit api_v4_AllTests ;;
    phpunit-civi)  task_phpunit 'Civi\AllTests' ;;
    phpunit-drupal) task_phpunit_drupal ;;
    phpunit-wordpress) task_phpunit_wordpress ;;
    phpunit-backdrop) task_phpunit_backdrop ;;
    phpunit-creditnotes) task_phpunit_core_extension sequentialcreditnotes ;;
    phpunit-flexmailer) task_phpunit_core_extension flexmailer ;;
    phpunit-financialacls) task_phpunit_core_extension financialacls ;;
    phpunit-eway) task_phpunit_core_extension ewaysingle ;;
    phpunit-afform) test_afform_extensions "$CIVI_CORE/ext/afform" ;;
    phpunit-authx) task_phpunit_core_extension authx ;;
    phpunit-core-exts) test_core_extenions ;;
    *) echo "unrecognized suite: $TESTTYPE"; exit 2 ;;
  esac
done

## Check test results and set exit code
echo "Check test results and set exit code"
cat "$JUNITDIR"/*.xml | grep '<failure' -q && fatal "Found <failure> in XML"
cat "$JUNITDIR"/*.xml | grep '<error' -q && fatal "Found <error> in XML"
for TESTTYPE in $SUITES ; do
  if  [ $TESTTYPE = "upgrade" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/civicrm-upgrade-test.xml" ] && fatal "Missing XML: civicrm-upgrade-test.xml"
  fi
  if  [ $TESTTYPE = "phpunit-crm" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/CRM_AllTests.xml" ] && fatal "Missing XML: CRM_AllTests.xml"
  fi
  if  [ $TESTTYPE = "phpunit-api" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/api_v3_AllTests.xml" ] && fatal "Missing XML: api_v3_AllTests.xml"
  fi
  if  [ $TESTTYPE = "phpunit-api3" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/api_v3_AllTests.xml" ] && fatal "Missing XML: api_v3_AllTests.xml"
  fi
  if  [ $TESTTYPE = "phpunit-api4" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/api_v4_AllTests.xml" ] && fatal "Missing XML: api_v4_AllTests.xml"
  fi
  if  [ $TESTTYPE = "phpunit-drupal" ] ; then
    [ ! -f "$JUNITDIR/phpunit_drupal.xml" ] && fatal "Missing XML: phpunit_drupal.xml"
  fi
  if  [ $TESTTYPE = "phpunit-wordpress" ] ; then
    [ ! -f "$JUNITDIR/phpunit_wordpress.xml" ] && fatal "Missing XML: phpunit_wordpress.xml"
  fi
  if  [ $TESTTYPE = "phpunit-backdrop" ] ; then
    [ ! -f "$JUNITDIR/phpunit_backdrop.xml" ] && fatal "Missing XML: phpunit_backdrop.xml"
  fi
  if  [ $TESTTYPE = "phpunit-civi" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/Civi\\AllTests.xml" ] && fatal "Missing XML: Civi\\AllTests.xml"
  fi
  if  [ $TESTTYPE = "phpunit-e2e" ] || [ $TESTTYPE = "all" ] ; then
    [ ! -f "$JUNITDIR/E2E_AllTests.xml" ] && fatal "Missing XML: E2E_AllTests.xml"
  fi
  if  [ $TESTTYPE = "karma" ] || [ $TESTTYPE = "all" ] ; then
    [ -d "$CIVI_CORE/tests/karma" -a ! -f "$JUNITDIR/karma.xml" ] && fatal "Missing XML: karma.xml"
  fi
done
[ -n "$EXITCODES" ] && fatal "At least one command failed abnormally [$EXITCODES]"
echo "Exit normally"