Add branch-based CI and Singularity image deployment

.travis.yml

@@ -1,71 +1,99 @@
-dist: xenial
-sudo: required
 language: cpp
 
+stages:
+  - name: basic
+    if: branch != master
+  - name: full linux
+    if: branch = master
+  - name: singularity build push
+    if: branch = master OR tag IS present
+    allow_failures: true
+
+env:
+  global:
+    - DISPLAY=":99.0"
+
 services:
   - xvfb
 
-matrix:
+jobs:
   include:
-    - os: linux
-      addons:
-        apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-6
-      env: COMPILER=gcc GCC=6
-    - os: linux
-      if: branch = master
+    - stage: basic
+      env: PYTHON_VERSION="3.7.5" COMPILER="gcc" GCCv="7"
+    - stage: full linux
+      os: linux
+      env: PYTHON_VERSION="3.7.5" COMPILER="gcc" GCCv="7"
+    - stage: full linux
+      os: linux
+      env: PYTHON_VERSION="3.7.5" COMPILER="gcc" GCCv="6"
+    - stage: full linux
+      os: linux
+      env: PYTHON_VERSION="3.6.9" COMPILER="gcc" GCCv="7"
+    - stage: full linux
+      os: linux
+      env: PYTHON_VERSION="3.6.9" COMPILER="gcc" GCCv="6"
+    - stage: singularity build push
+      env: PYTHON_VERSION="3.7.5"
+      language: go
+      go: "1.13"
+      git: { submodules: false, depth: 1 }
       addons:
         apt:
-          sources:
-            - ubuntu-toolchain-r-test
-          packages:
-            - g++-7
-      env: COMPILER=gcc GCC=7
-
-env:
-  global:
-    - DISPLAY=":99.0"
-    - MINCONDA_VERSION="latest"
-    - MINCONDA_LINUX="Linux-x86_64"
+          packages: [ flawfinder, squashfs-tools, uuid-dev, libuuid1, libffi-dev, libssl-dev, libssl1.0.0, libarchive-dev, libgpgme11-dev, libseccomp-dev ]
+        homebrew: { packages: [ squashfs ], update: true }
+      before_install: skip
+      install:
+        - SINGULARITY_BASE="${GOPATH}/src/github.com/sylabs/singularity"
+        - export PATH="${GOPATH}/bin:${PATH}"
+        - mkdir -p "${GOPATH}/src/github.com/sylabs"
+        - cd "${GOPATH}/src/github.com/sylabs"
+        - git clone -b v3.5.0 https://github.com/sylabs/singularity
+        - cd singularity
+        - ./mconfig -v -p /usr/local
+        - make -j `nproc 2>/dev/null || echo 1` -C ./builddir all
+        - sudo make -C ./builddir install
+      before_script:
+          #  token used for push commnad
+        - echo -e "$SYLABS_TOKEN" > token
+        - singularity remote login --tokenfile ./token || exit 1
+          #  key used for image signing
+          #  for travis ci pgp key block you must replace newline with '\\n', and replace spaces with '\ '
+        - echo -e "$SINGULARITY_KEY" > skey
+        - head -n 4 ./skey
+        - echo $SINGULARITY_KEY_PW | singularity key import ./skey || exit 1
+      script:
+        - cd $TRAVIS_BUILD_DIR
+        - export SINGULARITYENV_PYTHON_VERSION=$PYTHON_VERSION
+        - export SINGULARITYENV_GIT_COMMIT_HASH=$TRAVIS_COMMIT
+        - sudo singularity build ./extra-foam.sif extra-foam.def
+      after_script:
+        - echo $SINGULARITY_KEY_PW | singularity sign ./extra-foam.sif
+        - singularity push ./extra-foam.sif library://robert.rosca/default/extra-foam:$TRAVIS_BRANCH
 
 before_install:
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      if [[ "$COMPILER" == "gcc" ]]; then
-        export CXX=g++-$GCC CC=gcc-$GCC;
-      fi
-    fi
-
-install:
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      MINCONDA_OS=$MINCONDA_LINUX;
-    fi
-  - wget "http://repo.continuum.io/miniconda/Miniconda3-$MINCONDA_VERSION-$MINCONDA_OS.sh" -O miniconda.sh
+  - export GXX="g++-$GCCv" GCC="gcc-$GCCv"
+  - sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test"
+  - sudo apt-get -q update
+  - sudo -E apt-get -yq --no-install-suggests --no-install-recommends $(travis_apt_get_options) install libxkbcommon-x11-0 $GXX
+  - sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/$GXX 0
+  - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/$GCC 0
+  - g++ --version
+  - gcc --version
+  - wget "http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda.sh
   - bash miniconda.sh -b -p $HOME/miniconda
   - export PATH="$HOME/miniconda/bin:$PATH"
+install:
+  - conda install -y python=$PYTHON_VERSION
+  - echo $PYTHON_VERSION
   - conda install -y cmake -c conda-forge
-  - which python
-  - python --version
-  - which pip
-  - pip --version
-  # QT_DEBUG_PLUGINS
-  - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      sudo apt install -y libxkbcommon-x11-0;
-      pip install -e .[test];
-    fi
-
+  - pip install -e .[test]
 before_script:
   - /sbin/start-stop-daemon --start --quiet --pidfile /tmp/custom_xvfb_99.pid --make-pidfile --background --exec /usr/bin/Xvfb -- :99 -ac -screen 0 1400x900x24
   - sleep 3
-
 script:
   # test parallel version
   - python setup.py build_ext --with-tests
   - python setup.py test -v
-  - python setup.py benchmark -v
 
   # test series version
   - export FOAM_WITH_TBB=0
@@ -74,4 +102,3 @@ script:
   - export XTENSOR_WITH_XSIMD=0
   - python setup.py build_ext --with-tests
   - python setup.py test -v
-  - python setup.py benchmark -v

extra-foam.def

@@ -0,0 +1,87 @@
+Bootstrap: library
+From: centos:7.6.1810
+
+%help
+  EXtra-foam (previously known as karaboFAI) is an application that provides
+  super fast on-line (real-time) and off-line data analysis and visualization
+  for experiments at European XFEL that using 2D detectors, namely AGIPD, DSSC,
+  LPD, FastCCD, JungFrau, etc., together with other 1D detectors (e.g. XGM,
+  digitizer, etc.) and various control data.
+
+  GitHub: https://github.com/European-XFEL/EXtra-foam
+  Docs: https://extra-foam.readthedocs.io/en/latest/
+
+%environment
+  __conda_setup="$('/usr/local/bin/conda' 'shell.bash' 'hook' 2> /dev/null)"
+  if [ $? -eq 0 ]; then
+      eval "$__conda_setup"
+  else
+      if [ -f "/usr/local/etc/profile.d/conda.sh" ]; then
+          . "/usr/local/etc/profile.d/conda.sh"
+      else
+          export PATH="/usr/local/bin:$PATH"
+      fi
+  fi
+  unset __conda_setup
+
+  conda activate base
+
+%runscript
+  extra-foam "$@"
+
+%post
+  export logpath=/.singularity.d/logs/
+  mkdir -p $logpath
+
+  #  Set default python version if missing
+  PYTHON_VERSION="${PYTHON_VERSION:-3.7.5}"
+  echo "Building for python version: $PYTHON_VERSION"
+
+  #  Basic yum dependencies
+  ##  Install and update
+  yum update -y | tee $logpath/00-yum-update.log
+  yum install -y epel-release | tee $logpath/01-yum-epel-release.log
+  yum groupinstall -y 'Development Tools' | tee $logpath/02-yum-dev-tools.log
+  yum install -y      \
+    nano              \
+    curl              \
+    wget              \
+    tar               \
+    bzip2             \
+    git               \
+    e4fsprogs         \
+    xeyes             \
+    mesa-libGL        \
+    qt5-qtbase        \
+    libxkbcommon-x11  \
+  | tee $logpath/03-yum-install.log
+
+  ##  Log yum packages
+  yum list installed | tee $logpath/04-yum-list-installed.log
+
+  #  Install and setup miniconda
+  ##  Download and install minconda
+  curl -sSL http://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o /tmp/miniconda.sh
+  chmod +x /tmp/miniconda.sh
+  /tmp/miniconda.sh -bfp /usr/local/
+  rm -f /tmp/miniconda.sh
+  source /usr/local/bin/activate
+
+  ##  Install python dependencies
+  conda install -y python=$PYTHON_VERSION
+  conda install -y -c anaconda cmake numpy
+  conda install -y -c omgarcia gcc-6
+
+  ##  Download EXtra-foam source
+  mkdir -p /usr/local/src/
+  cd /usr/local/src/
+  git clone https://github.com/RobertRosca/EXtra-foam
+  cd EXtra-foam
+  git checkout $GIT_COMMIT_HASH
+  git submodule update --init | tee -a $logpath/05-extra-foam-submodules.log
+  git log -n 1 | tee -a $logpath/06-extra-foam-clone.log
+
+  pip install .
+
+  ##  Export environment to concretise
+  conda env export -n base -f /.singularity.d/logs/07-conda-env-export.log