Improve text filter in compare_results.sh

.travis.yml

@@ -17,13 +17,302 @@ jobs:
     - name: "openFOAM adapter [PETSc] [Job failure permitted]"
     - name: "CalculiX adapter [PETSc] [Job failure permitted]"
     - name: "[16.04 PETSc] OpenFOAM <-> CalculiX [FSI] [Job failure permitted]"
-      name: "[16.04] SU2 <-> Calculix [unstable][Job failure permitted]"
+    - name: "[16.04] SU2 <-> Calculix [unstable][Job failure permitted]"
     - name: "[18.04] nutils <-> OpenFOAM [Job failure permitted]"
 
-
   include:
+    - stage: Building preCICE
+      name: "Arch Linux"
+      if: fork = false
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Arch --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Arch
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Arch --docker-username $DOCKER_USERNAME
+
+    - stage: Building preCICE
+      name: "Ubuntu 16.04 home"
+      if: fork = false
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Ubuntu1604.home --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Ubuntu1604.home
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Ubuntu1604.home --docker-username $DOCKER_USERNAME
+
+    - stage: Building preCICE
+      name: "Ubuntu 16.04 home PETSc [Job failure permitted]"
+      if: fork = false
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Ubuntu1604.home --petsc yes --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Ubuntu1604.home
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Ubuntu1604.home --petsc yes --docker-username $DOCKER_USERNAME
+
+    - stage: Building preCICE
+      name: "Ubuntu 16.04.sudo"
+      if: type = cron
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Ubuntu1604.sudo --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Ubuntu1604.sudo
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Ubuntu1604.sudo --docker-username $DOCKER_USERNAME
+
+    - stage: Building preCICE
+      name: "Ubuntu 18.04.home"
+      if: fork = false
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.home --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Ubuntu1804.home
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.home --docker-username $DOCKER_USERNAME
+
+    - stage: Building preCICE
+      name: "Ubuntu 18.04.sudo"
+      if: type = cron
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.sudo --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Ubuntu1804.sudo
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.sudo --docker-username $DOCKER_USERNAME
+
+    - stage: Building preCICE
+      name: "Ubuntu 18.04.sudo.mpich [Job failure permitted]"
+      if: type = cron
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.sudo.mpich --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Ubuntu1804.sudo.mpich
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.sudo.mpich --docker-username $DOCKER_USERNAME
+
+    - stage: Building preCICE
+      name: "Ubuntu 18.04.package"
+      if: fork = false
+      script:
+        - python build_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.package --docker-username $DOCKER_USERNAME
+        - python push.py --precice precice/Dockerfile.Ubuntu1804.package
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_precice.py --dockerfile precice/Dockerfile.Ubuntu1804.package --docker-username $DOCKER_USERNAME
+
+    - name: "[Build preCICE on fork] Using cached version"
+      if: fork = true
+      script: true
+
+
+
+    - stage: Building adapters
+      name: "[16.04] SU2 adapter"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.su2-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.su2-adapter
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.su2-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+
+    - name: "[16.04] CalculiX adapter"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.calculix-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.calculix-adapter
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.calculix-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+
+    - name: "[16.04] deal.ii adapter"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.dealii-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.dealii-adapter
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.dealii-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+
+    - name: "[16.04] OpenFOAM adapter"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.openfoam-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.openfoam-adapter
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.openfoam-adapter --operating-system ubuntu1604 --precice-installation home --docker-username $DOCKER_USERNAME
+
+    - name: "[18.04] OpenFOAM adapter"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.openfoam-adapter.Ubuntu1804 --operating-system ubuntu1804 --precice-installation home --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.openfoam-adapter.Ubuntu1804
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.openfoam-adapter.Ubuntu1804 --operating-system ubuntu1804 --precice-installation home --docker-username $DOCKER_USERNAME
+
+    - name: "[18.04] FEniCS adapter"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.fenics-adapter --operating-system ubuntu1804 --precice-installation home --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.fenics-adapter
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.fenics-adapter --operating-system ubuntu1804 --precice-installation home --docker-username $DOCKER_USERNAME
+
+    - name: "[16.04] OpenFOAM adapter [PETSc] [Job failure permitted]"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.openfoam-adapter --operating-system ubuntu1604 --precice-installation home --petsc yes --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.openfoam-adapter
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.openfoam-adapter --operating-system ubuntu1604 --precice-installation home --petsc yes --docker-username $DOCKER_USERNAME
+
+    - name: "[16.04] CalculiX adapter [PETSc] [Job failure permitted]"
+      if: fork = false
+      script:
+        - python build_adapter.py --dockerfile adapters/Dockerfile.calculix-adapter --operating-system ubuntu1604 --precice-installation home --petsc yes --docker-username $DOCKER_USERNAME
+        - python push.py --adapter adapters/Dockerfile.calculix-adapter
+      deploy:
+        skip_cleanup: true
+        provider: script
+        on:
+          all_branches: true
+        script: >-
+          echo $DOCKER_PASSWORD | docker login -u $DOCKER_USERNAME --password-stdin &&
+          python push_adapter.py --dockerfile adapters/Dockerfile.calculix-adapter --operating-system ubuntu1604 --precice-installation home --petsc yes --docker-username $DOCKER_USERNAME
+
+    - name: "[Build adapters on fork] Using cached version"
+      if: fork = true
+      script: true
+
+
+
     - stage: Tests
-      name: "[16.04] Elastictube1D - Python"
+      name: "[16.04] SU2 <-> Calculix [unstable][Job failure permitted]"
+      script:
+        - python system_testing.py -s su2-ccx -v
+        - python push.py --test su2-ccx
+
+    - name: "[16.04] OpenFOAM <-> OpenFOAM"
+      script:
+        - python system_testing.py -s of-of -v
+        - python push.py --test of-of
+
+    - name: "[16.04] Calculix <-> OpenFOAM"
+      script:
+        - python system_testing.py -s of-ccx -v
+        - python push.py --test of-ccx
+
+    - name: "[18.04] FEniCS <-> FEniCS"
+      script:
+        - python system_testing.py -s fe-fe --base Ubuntu1804.home -v
+        - python push.py --test fe-fe --base Ubuntu1804.home
+
+    - name: "[16.04] Bindings/Solverdummies"
+      script:
+        - python system_testing.py -s bindings -v
+        - python push.py --test bindings
+
+    - name: "[18.04] Bindings/Solverdummies"
+      script:
+        - python system_testing.py -s bindings --base Ubuntu1804.home -v
+        - python push.py --test bindings --base Ubuntu1804.home
+
+    - name: "[16.04] deal.ii <-> OpenFOAM"
+      script:
+        - python system_testing.py -s dealii-of -v
+        - python push.py --test dealii-of
+
+    - name: "[18.04] nutils <-> OpenFOAM [Job failure permitted]"
+      script:
+        - python system_testing.py -s nutils-of --base Ubuntu1804.home -v
+        - python push.py --test nutils-of --base Ubuntu1804.home
+
+    - name: "[16.04] OpenFOAM <-> OpenFOAM [nearest projection]"
+      script:
+        - python system_testing.py -s of-of_np -v
+        - python push.py --test of-of_np
+
+    - name: "[16.04] Elastictube1D - Python"
       script:
         - python system_testing.py -s 1dtube_py -v
         - python push.py --test 1dtube_py
@@ -32,3 +321,8 @@ jobs:
       script:
         - python system_testing.py -s 1dtube_cxx -v
         - python push.py --test 1dtube_cxx
+
+    - name: "[16.04 PETSc] OpenFOAM <-> CalculiX [FSI] [Job failure permitted]"
+      script:
+        - python system_testing.py -s of-ccx_fsi --base Ubuntu1604.home.PETSc -v
+        - python push.py --test of-ccx_fsi --petsc

compare_results.sh

@@ -55,22 +55,34 @@ if [ -n "$diff_files" ]; then
   mapfile -t array_files < <( echo "$diff_files"  | sed 's/Files\|and\|differ//g' )
   arr_len="${#array_files[@]}"
 
+  # loop through differing files
   for (( i = 0; i<${arr_len}; i = i + 1));
   do
     file1=$( echo "${array_files[i]}" | awk '{print $1}' )
     file2=$( echo "${array_files[i]}" | awk '{print $2}' )
 
-    raw_diff=$( diff -y --speed-large-files --suppress-common-lines "$file1" "$file2" )
     # Filter output files, ignore lines with words (probably not the results)
-    # removes |<>() characters
     # Do not delete "e", since it can be used as exponent
-    num_diff=$( echo "$raw_diff" | sed 's/(\|)\||\|>\|<//g; /[a-df-zA-Z]\|Version/d' )
+    # removes |<>() characters
+    num_filter='s/(\|)\||\|>\|<//g; /[a-df-zA-Z]\|[vV]ersion/d'
     # Filter for text lines. Compare these seperately from numerical lines
-    text_diff=$( echo "$raw_diff" | sed '/[A-Za-df-z]/!d' )
+    # Ignore any timestamps
+    txt_filter='s/(\|)\||\|>\|<//g; /[a-df-zA-Z]\|[vV]ersion/!d; s/[0-9][0-9]:[0-9][0-9]:[0-9][0-9]//g; /Timestamp\|[rR]untime\|Unexpected end of/d; /Run finished/q'
+    file1_num=$( cat "$file1" | sed "$num_filter")
+    file2_num=$( cat "$file2" | sed "$num_filter")
+
+    file1_txt=$( cat "$file1" | sed "$txt_filter")
+    file2_txt=$( cat "$file2" | sed "$txt_filter")
+
+
+    num_diff=$( diff -y --speed-large-files --suppress-common-lines <(echo "$file1_num") <(echo "$file2_num") )
+    txt_diff=$( diff -y --speed-large-files --suppress-common-lines <(echo "$file1_txt") <(echo "$file2_txt") )
 
 
     # Pairwise compares files fields, that are produces from diffs and computes average and maximum
     # relative differences
+    filename=$(basename $file1) # total file paths are pretty long, this keep info concise
+    echo "Comparing values in '$filename'..."
     if [ -n "$num_diff" ]; then
       rel_max_difference=$( export max_diff_limit; export avg_diff_limit; echo "$num_diff" | awk 'function abs(v) {return v < 0 ? -v : v} { radius=NF/2;
               max_diff=0;
@@ -89,12 +101,13 @@ if [ -n "$diff_files" ]; then
       # Split by space and transform into the array
       difference=( $rel_max_difference )
       echo -e "> Numerical difference in $file1 and $file2"
+      echo -e "$num_diff"
       echo -e "Average: ${difference[0]} ; Maximum: ${difference[1]} ${NC}"
       ret=1
     fi
-    if [ -n "$text_diff" ]; then
+    if [ -n "$txt_diff" ]; then
       echo -e "> Text difference in $file1 and $file2"
-      echo -e "$text_diff"
+      echo -e "$txt_diff"
       ret=1
     fi
   done

system_testing.py

@@ -162,7 +162,7 @@ def comparison(pathToRef, pathToOutput):
 def build_run_compare(test, tag, branch, local_precice, force_rebuild, rm_all=False, verbose=False):
     """ Runs and compares test, using precice branch. """
     compose_tests = ["dealii-of", "of-of", "su2-ccx", "of-ccx", "of-of_np",
-            "fe-fe","nutils-of", "of-ccx_fsi", "1dtube_cxx"]
+            "fe-fe","nutils-of", "of-ccx_fsi", "1dtube_cxx", "1dtube_py"]
     test_basename = test.split(".")[0]
     if local_precice:
         build_adapters(test_basename, tag, branch, local_precice, force_rebuild)
@@ -225,7 +225,7 @@ def compose_tag(docker_username, base, features, branch):
     parser.add_argument('-l', '--local', action='store_true', help="Use local preCICE image (default: use remote image)")
     parser.add_argument('-s', '--systemtest', type=str, help="Choose system tests you want to use",
                         choices = common.get_tests(), required = True)
-    parser.add_argument('-b', '--branch', help="preCICE branch to use", default=os.environ["TRAVIS_BRANCH"] if os.environ["TRAVIS_PULL_REQUEST"] == "false" else "develop")  # make sure that branch corresponding to system tests branch is used, if no branch is explicitly specified. If we are testing a pull request, will test against develop by default.
+    parser.add_argument('-b', '--branch', help="preCICE branch to use", default="develop")  # make sure that branch corresponding to system tests branch is used, if no branch is explicitly specified. If we are testing a pull request, will test against develop by default.
 # Usage of the branch argument:
 #   When on a PR, this will by default use the develop versions of preCICE and adapter images.
 #   This makes it easier to experiment with tests, which are most commonly addressed by PRs

tests/TestCompose_1dtube_cxx.Ubuntu1604.home/docker-compose.yml

@@ -12,8 +12,9 @@ services:
       - output:/home/precice/Data/Output/
     command: >
       /bin/bash -c "./Allrun &&
-        cp *.json *.log /home/precice/Data/Output/ &&
-        cp -r Postproc /home/precice/Data/Output/"
+        cp *.log /home/precice/Data/Output/ &&
+        cp -r Postproc /home/precice/Data/Output/ &&
+        rm /home/precice/Data/Output/*summary.log"
     container_name: 1dtube-ccx
 
   tutorial-data: