Merge pull request #11 from ICRAR/devel

Devel

.appveyor.yml

@@ -35,8 +35,6 @@ install:
   - cmd: git submodule update --init --recursive
   - cmd: git clone https://github.com/CxxTest/cxxtest
   - cmd: cd c:\tools\vcpkg
-  - cmd: git pull
-  - cmd: vcpkg update
   - cmd: vcpkg integrate install
   - cmd: vcpkg install boost-filesystem:x64-windows
   - cmd: vcpkg install boost-log:x64-windows
@@ -60,9 +58,12 @@ test_script:
   - ps: Invoke-WebRequest "https://docs.google.com/uc?export=download&id=1JDK8ak13bEhzg9H9xt0uE8Fh_2LD3KpZ" -OutFile "input\tree_199.0.hdf5"
   - cmd: shark sample.cfg -o simulation.redshift_file=input\redshifts.txt -o simulation.tree_files_prefix=input\tree_199 -o execution.seed=123456 -o execution.name_model=my_model
   - cmd: C:\Python36-x64\Scripts\pip install h5py matplotlib scipy
-  - cmd: C:\Python36-x64\python.exe standard_plots\all.py -c sample.cfg -z input\redshifts.txt
+  # Plots are currently broken
+  #- cmd: C:\Python36-x64\python.exe standard_plots\all.py -c sample.cfg -z input\redshifts.txt
   - cmd: shark sample.cfg -o simulation.redshift_file=input\redshifts.txt -o simulation.tree_files_prefix=input\tree_199 -o execution.seed=123456 -o execution.name_model=my_model_equal_seed
   - cmd: C:\Python36-x64\python.exe scripts\compare_galaxies.py -m mini-SURFS\my_model\199\0\galaxies.hdf5 mini-SURFS\my_model_equal_seed\199\0\galaxies.hdf5
+  - cmd: shark sample.cfg -o simulation.redshift_file=input\redshifts.txt -o simulation.tree_files_prefix=input\tree_199 -o execution.seed=123456 -o execution.name_model=my_model_equal_seed_parallel -t 2
+  - cmd: C:\Python36-x64\python.exe scripts\compare_galaxies.py -m mini-SURFS\my_model\199\0\galaxies.hdf5 mini-SURFS\my_model_equal_seed_parallel\199\0\galaxies.hdf5
   - cmd: shark sample.cfg -o simulation.redshift_file=input\redshifts.txt -o simulation.tree_files_prefix=input\tree_199 -o execution.name_model=my_model_random_seed
   - cmd: C:\Python36-x64\python.exe scripts\compare_galaxies.py -m mini-SURFS\my_model_random_seed\199\0\galaxies.hdf5 mini-SURFS\my_model_equal_seed\199\0\galaxies.hdf5 --expect-unequal
 

.github/workflows/build-and-test.yaml

@@ -0,0 +1,40 @@
+name: Build and test
+
+# Build on every branch push, tag push, and pull request change:
+on: [push]
+
+jobs:
+  build_wheels:
+    name: Build and test shark on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: Install dependencies
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt install libhdf5-dev hdf5-tools libboost-filesystem-dev libboost-program-options-dev libboost-log-dev cxxtest libgsl-dev
+
+      - name: Install dependencies
+        if: ${{ matrix.os == 'macos-latest' }}
+        run: |
+          brew install hdf5 boost cxxtest gsl
+
+      - name: Configure
+        # Leaving Werror out for now because there *are* errors
+        # and I don't know what the proper fix is.
+        run: cmake -B build/ -DSHARK_TEST=ON -DCMAKE_CXX_FLAGS="-Wall" #-Werror
+
+      - name: Build
+        run: cmake --build build/
+
+      - name: Run unit tests
+        run: |
+          cd build
+          ctest --output-on-failure

.readthedocs.yaml

@@ -0,0 +1,8 @@
+version: 2
+
+submodules:
+  include: all
+
+sphinx:
+  configuration: doc/conf.py
+  fail_on_warning: true

.travis.yml → .travis-unused.yaml

@@ -27,49 +27,48 @@
 language: cpp
 sudo: false
 
-# In Linux we cache the gsl binary version
-cache:
- directories:
-  - gsl/2.4
-
 # We try with both GNU and LLVM
 # When running with gcc-6 we also collect coverage information
 # and upload it to coveralls
 matrix:
  include:
   - compiler: gcc
+    dist: xenial
     addons:
      apt:
       sources: ubuntu-toolchain-r-test
-      packages: [g++-5, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest]
+      packages: [g++-5, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest, libgsl-dev]
     env: COMPILER=g++-5
   - compiler: gcc
+    dist: xenial
     addons:
      apt:
       sources: ubuntu-toolchain-r-test
-      packages: [g++-6, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest, python-matplotlib, python-scipy, python-h5py, python-tk]
+      packages: [g++-6, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest, libgsl-dev, python-matplotlib, python-scipy, python-h5py, python-tk]
     env: COMPILER=g++-6 PYTHON=python
   - compiler: gcc
+    dist: xenial
     addons:
      apt:
       sources: ubuntu-toolchain-r-test
-      packages: [g++-7, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest, python3-matplotlib, python3-scipy, python3-h5py, python3-tk]
+      packages: [g++-7, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest, libgsl-dev, python3-matplotlib, python3-scipy, python3-h5py, python3-tk]
     env: COMPILER=g++-7 PYTHON=python3
   - compiler: clang
+    dist: xenial
     addons:
      apt:
       sources: ubuntu-toolchain-r-test
-      packages: [g++-7, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest]
-    env: COMPILER=clang++ LD_LIBRARY_PATH=/usr/local/clang-5.0.0/lib
+      packages: [g++-7, libhdf5-dev, hdf5-tools, libboost-filesystem-dev, libboost-program-options-dev, libboost-log-dev, cxxtest, libgsl-dev]
+    env: COMPILER=clang++ LD_LIBRARY_PATH=/usr/local/clang-7.0.0/lib
   - os: osx
-    osx_image: xcode8.3 # OSX 10.12
-    env: XCODE=8.3
+    osx_image: xcode12.2 # OSX 10.15.7
+    env: COMPILER=clang++ PYTHON=venv
   - os: osx
-    osx_image: xcode9.1 # OSX 10.12
-    env: XCODE=9.1 PYTHON=venv
+    osx_image: xcode13.1  # OSX 11.6
+    env: COMPILER=clang++
   - os: osx
-    osx_image: xcode10  # OSX 10.13
-    env: XCODE=10
+    osx_image: xcode13.3  # OSX 12.3
+    env: COMPILER=clang++
 
 # We "source" it because it exports variables
 before_install:

.travis/before_install.sh

@@ -36,21 +36,15 @@ cd ${TRAVIS_BUILD_DIR}
 if [ "${TRAVIS_OS_NAME}" = "osx" ]
 then
 
-	# cxxtest pulls python@2, so we need to unlink
-	# the pre-installed python first
-	brew unlink python || fail "cannot unlink python"
+	export HOMEBREW_NO_AUTO_UPDATE=1
+	export HOMEBREW_NO_INSTALL_CLEANUP=1
 
-	# Minimal dependencies for testing
-	pkgs="gsl hdf5 cxxtest"
-	if [ "$PYTHON" = "venv" ]
-	then
-		pkgs="$pkgs python3"
-	fi
+	# cxxtest pulls python, so we need to unlink the pre-installed python@2 first
+	# (only in xcode12.2 though, it doesn't come installed in the rest)
+	test "${TRAVIS_OSX_IMAGE}" != xcode12.2 || brew unlink python@2 || fail "cannot unlink python@2"
 
-	# "install" seems to be "update"-ing too, which is failing
-	# with a "don't worry, [...] everything is [...] fine now"
-	# message. Let's follow that advice more explicitly
-	brew update || true
+	# Minimal dependencies for testing
+	pkgs="gsl cxxtest libomp"
 	brew install $pkgs || fail "cannot install packages: $pkgs"
 
 	# PYTHON==venv means that we are going to use a virtualenv'd python
@@ -60,26 +54,10 @@ then
 	then
 		python3 -mvenv "${TRAVIS_BUILD_DIR}/shark-venv" || fail "cannot create virtualenv"
 		source "${TRAVIS_BUILD_DIR}/shark-venv/bin/activate"
-		pip install -U pip wheel setuptools h5py matplotlib scipy || fail "cannot install python packages in virtualenv"
+		pip install -U pip wheel setuptools h5py matplotlib!=3.4 scipy || fail "cannot install python packages in virtualenv"
 		deactivate
 		PYTHON="${TRAVIS_BUILD_DIR}/shark-venv/bin/python3"
 	fi
 
 	return
 fi
-
-# Ubuntu Travis still comes with GSL 1.X but we need >= 2
-# We cache the binary version through travis' cache, so let's
-# check first if it exists
-export GSL_ROOT_DIR=${TRAVIS_BUILD_DIR}/gsl/2.4
-export LD_LIBRARY_PATH=${GSL_ROOT_DIR}/lib:$LD_LIBRARY_PATH
-if [ ! -d "${GSL_ROOT_DIR}/lib" ]
-then
-	curl -O https://mirror.freedif.org/GNU/gsl/gsl-2.4.tar.gz
-	tar xf gsl-2.4.tar.gz
-	cd gsl-2.4
-	./configure --prefix=${GSL_ROOT_DIR}
-	make all -j2
-	make install
-	cd ..
-fi

.travis/install.sh

@@ -34,9 +34,15 @@ cd ${TRAVIS_BUILD_DIR}
 mkdir build
 cd build
 
-SHARK_CMAKE_OPTIONS="-DCMAKE_CXX_COMPILER=$COMPILER -DSHARK_TEST=ON -DGSL_ROOT_DIR=${GSL_ROOT_DIR}"
+SHARK_CMAKE_OPTIONS="-DCMAKE_CXX_COMPILER=$COMPILER -DSHARK_TEST=ON -DCMAKE_CXX_FLAGS=\"-Wall -Werror\""
+if [ ${TRAVIS_OS_NAME} = osx ]; then
+	SHARK_CMAKE_OPTIONS+=" -DOpenMP_CXX_LIB_NAMES=libomp"
+	SHARK_CMAKE_OPTIONS+=" -DOpenMP_CXX_FLAGS='-Xpreprocessor -fopenmp -I/usr/local/include'"
+	SHARK_CMAKE_OPTIONS+=" -DOpenMP_libomp_LIBRARY=/usr/local/lib/libomp.dylib"
+	SHARK_CMAKE_OPTIONS+=" -DCMAKE_CXX_FLAGS=-Wno-unused-command-line-argument"
+fi
 
 # Go, go, go!
-cmake .. ${SHARK_CMAKE_OPTIONS} || fail "cmake failed"
+eval cmake .. ${SHARK_CMAKE_OPTIONS} || fail "cmake failed"
 make all -j2 || fail "make failed"
 cd ..

.travis/test.sh

@@ -40,11 +40,15 @@ mkdir input || fail "failed to create input/ directory"
 curl -L -o input/redshifts.txt 'https://docs.google.com/uc?export=download&id=1xvNmJB_KmoBHuQz-QzdPnY0HFs7smkUB' || fail "failed to download redshifts file"
 curl -L -o input/tree_199.0.hdf5 'https://docs.google.com/uc?export=download&id=1JDK8ak13bEhzg9H9xt0uE8Fh_2LD3KpZ' || fail "failed to download test hdf5 file"
 
-./shark ../sample.cfg \
-    -o simulation.redshift_file=input/redshifts.txt \
-    -o simulation.tree_files_prefix=input/tree_199 \
-    -o execution.seed=123456 \
-    -o execution.name_model=my_model || fail "failure during execution of shark"
+run_shark() {
+	model_name=$1; shift
+	./shark ../sample.cfg \
+	    -o simulation.redshift_file=input/redshifts.txt \
+	    -o simulation.tree_files_prefix=input/tree_199 \
+	    -o execution.name_model=$model_name $@ || fail "failure during execution of shark"
+}
+
+run_shark my_model -o execution.seed=123456
 
 # Generate the HDF5 output documentation and check it's up to date
 # otherwise tell the user how to update it
@@ -66,33 +70,33 @@ check_hdf5_doc 156/0/star_formation_histories.hdf5 star_formation_histories.rst
 
 if [ -n "$PYTHON" ]; then
 
+	# How many CPUs do we have?
+	if [ $TRAVIS_OS_NAME = osx ]; then
+		n_cpus=`sysctl -n hw.ncpu`
+	else
+		n_cpus=`grep -c processor /proc/cpuinfo`
+	fi
+
+	compare_galaxies() {
+		model_name=$1; shift
+		"$PYTHON" ../scripts/compare_galaxies.py \
+		    -m "mini-SURFS/my_model/199/0/galaxies.hdf5" \
+		       "mini-SURFS/$model_name/199/0/galaxies.hdf5" $@
+	}
+
 	# Make sure the standard plotting scripts run correctly
 	echo "backend: Agg" >> matplotlibrc
 	"$PYTHON" ../standard_plots/all.py -c ../sample.cfg -z input/redshifts.txt || fail "failure during execution of python plotting scripts"
 
 	# Make sure the seed value returns a reproducible result
-	./shark ../sample.cfg \
-	    -o simulation.redshift_file=input/redshifts.txt \
-	    -o simulation.tree_files_prefix=input/tree_199 \
-	    -o execution.seed=123456 \
-	    -o execution.name_model=my_model_equal_seed \
-	    || fail "failure during execution of shark"
+	run_shark my_model_same_seed -o execution.seed=123456
+	compare_galaxies my_model_same_seed || fail "Models expected to be equal, they are not."
 
-	"$PYTHON" ../scripts/compare_galaxies.py \
-	    -m "mini-SURFS/my_model/199/0/galaxies.hdf5" \
-	       "mini-SURFS/my_model_equal_seed/199/0/galaxies.hdf5" \
-	    || fail "Models expected to be equal, they are not."
+	# Like above, but in parallel using all available CPUs
+	run_shark my_model_same_seed_parallel -o execution.seed=123456 -t $n_cpus
+	compare_galaxies my_model_same_seed_parallel || fail "Models expected to be equal, they are not."
 
 	# Run using a random seed in the interval 2^32 - 1
-	./shark ../sample.cfg \
-	    -o simulation.redshift_file=input/redshifts.txt \
-	    -o simulation.tree_files_prefix=input/tree_199 \
-	    -o execution.name_model=my_model_random_seed \
-	    || fail "failure during execution of shark"
-
-	"$PYTHON" ../scripts/compare_galaxies.py \
-	    -m "mini-SURFS/my_model/199/0/galaxies.hdf5" \
-	       "mini-SURFS/my_model_random_seed/199/0/galaxies.hdf5" \
-	    --expect-unequal \
-	    || fail "Models expected to be unequal, they are not."
+	run_shark my_model_random_seed || fail "failure during execution of shark"
+	compare_galaxies my_model_random_seed --expect-unequal || fail "Models expected to be unequal, they are not."
 fi