Skip to content

Commit

Permalink
Merge branch 'master' into feature/10855_rewrite_indirect_diffraction…
Browse files Browse the repository at this point in the history
…_reduction

Refs #10855
  • Loading branch information
DanNixon committed May 15, 2015
2 parents 11748c8 + 3e5ef11 commit f93593a
Show file tree
Hide file tree
Showing 1,188 changed files with 51,973 additions and 16,833 deletions.
15 changes: 10 additions & 5 deletions Code/Mantid/Build/CMake/DarwinSetup.cmake
Expand Up @@ -107,11 +107,16 @@ set ( CMAKE_INSTALL_PREFIX "" )
set ( CPACK_PACKAGE_EXECUTABLES MantidPlot )
set ( INBUNDLE MantidPlot.app/ )

# We know exactly where this has to be on Darwin
set ( PARAVIEW_APP_DIR "/Applications/${OSX_PARAVIEW_APP}" )
set ( PARAVIEW_APP_BIN_DIR "${PARAVIEW_APP_DIR}/Contents/MacOS" )
set ( PARAVIEW_APP_LIB_DIR "${PARAVIEW_APP_DIR}/Contents/Libraries" )
set ( PARAVIEW_APP_PLUGIN_DIR "${PARAVIEW_APP_DIR}/Contents/Plugins" )
# We know exactly where this has to be on Darwin, but separate whether we have
# kit build or a regular build.
if ( ENABLE_CPACK AND MAKE_VATES )
add_definitions(-DBUNDLE_PARAVIEW)
else ()
set ( PARAVIEW_APP_DIR "${ParaView_DIR}" )
set ( PARAVIEW_APP_BIN_DIR "${PARAVIEW_APP_DIR}/bin" )
set ( PARAVIEW_APP_LIB_DIR "${PARAVIEW_APP_DIR}/lib" )
set ( PARAVIEW_APP_PLUGIN_DIR "${PARAVIEW_APP_DIR}/lib" )
endif ()

set ( BIN_DIR MantidPlot.app/Contents/MacOS )
set ( LIB_DIR MantidPlot.app/Contents/MacOS )
Expand Down
1,293 changes: 1,293 additions & 0 deletions Code/Mantid/Build/CMake/FindBoost.cmake

Large diffs are not rendered by default.

10 changes: 10 additions & 0 deletions Code/Mantid/Build/CMake/GNUSetup.cmake
Expand Up @@ -37,9 +37,19 @@ endif()
set( CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -fno-omit-frame-pointer" )
set( CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-omit-frame-pointer" )

option(WITH_ASAN "Enable address sanitizer" OFF)
if(WITH_ASAN)
message(STATUS "enabling address sanitizer")
add_compile_options(-fno-omit-frame-pointer -fno-common -fsanitize=address)
set(CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -fsanitize=address -lasan" )
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -lasan" )
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address -lasan" )
endif()

# Set the options for gcc and g++
set( CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${GNUFLAGS}" )
# -Wno-overloaded-virtual is down here because it's not applicable to the C_FLAGS
set( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${GNUFLAGS} -Woverloaded-virtual -fno-operator-names -std=c++0x" )

# Cleanup
set ( GNUFLAGS )
16 changes: 8 additions & 8 deletions Code/Mantid/Build/CMake/LinuxPackageScripts.cmake
Expand Up @@ -82,11 +82,9 @@ set ( PRE_UNINSTALL_FILE ${CMAKE_CURRENT_BINARY_DIR}/prerm )
set ( POST_UNINSTALL_FILE ${CMAKE_CURRENT_BINARY_DIR}/postrm )

if ( "${UNIX_DIST}" MATCHES "RedHatEnterprise" OR "${UNIX_DIST}" MATCHES "^Fedora" ) # RHEL/Fedora
if ( "${UNIX_CODENAME}" MATCHES "Santiago" OR
"${UNIX_CODENAME}" MATCHES "Maipo" )
set ( WRAPPER_PREFIX "scl enable mantidlibs \"" )
if ( "${UNIX_CODENAME}" MATCHES "Santiago" )
set ( WRAPPER_PREFIX "scl enable mantidlibs34 \"" )
set ( WRAPPER_POSTFIX "\"" )
set ( EXTRA_LDPATH "/usr/lib64/paraview" )
else()
set ( WRAPPER_PREFIX "" )
set ( WRAPPER_POSTFIX "" )
Expand Down Expand Up @@ -130,29 +128,31 @@ endif()
# MantidPlot launcher script
############################################################################
# Local dev version
set ( EXTRA_LDPATH "${ParaView_DIR}/lib" )
set ( MANTIDPLOT_EXEC MantidPlot )
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/launch_mantidplot.sh.in
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/launch_mantidplot.sh.in
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/launch_mantidplot.sh @ONLY )
# Needs to be executable
execute_process ( COMMAND "chmod" "+x" "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/launch_mantidplot.sh"
OUTPUT_QUIET ERROR_QUIET )
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/mantidpython.in
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/mantidpython.in
${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mantidpython @ONLY )
# Needs to be executable
execute_process ( COMMAND "chmod" "+x" "${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/mantidpython"
OUTPUT_QUIET ERROR_QUIET )

# Package version
set ( EXTRA_LDPATH "\${INSTALLDIR}/../lib/paraview-4.3" )
set ( MANTIDPLOT_EXEC MantidPlot_exe )
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/launch_mantidplot.sh.in
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/launch_mantidplot.sh.in
${CMAKE_CURRENT_BINARY_DIR}/launch_mantidplot.sh.install @ONLY )
install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/launch_mantidplot.sh.install
DESTINATION ${BIN_DIR} RENAME launch_mantidplot.sh
PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
GROUP_EXECUTE GROUP_READ
WORLD_EXECUTE WORLD_READ
)
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/mantidpython.in
configure_file ( ${CMAKE_MODULE_PATH}/Packaging/mantidpython.in
${CMAKE_CURRENT_BINARY_DIR}/mantidpython.install @ONLY )
install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/mantidpython.install
DESTINATION ${BIN_DIR} RENAME mantidpython
Expand Down
9 changes: 6 additions & 3 deletions Code/Mantid/Build/CMake/MPISetup.cmake
Expand Up @@ -16,18 +16,21 @@ set ( CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}" ${MPI_CXX_COMPILE_FLAGS} )
#set ( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}" ${MPI_CXX_LINK_FLAGS} )

set ( BOOST_ROOT /usr/lib64/openmpi ) # This is where (boost-)openmpi winds up on rhel6
find_package ( Boost REQUIRED mpi serialization )
find_package ( Boost COMPONENTS mpi serialization REQUIRED )
include_directories( ${Boost_INCLUDE_DIRS} )

#unset ( BOOST_ROOT )

# Add a definition that's used to guard MPI-specific parts of the main code
add_definitions ( -DMPI_BUILD )

add_subdirectory( ${CMAKE_CURRENT_SOURCE_DIR}/Testing/SystemTests/scripts )

# Add the ability to build a 'mantid-mpi' rpm
set ( CPACK_PACKAGE_NAME mantid-mpi )
include ( CPackCommon )
include ( CPackLinuxSetup )
# Framework dependencies
set ( CPACK_RPM_PACKAGE_REQUIRES "boost >= 1.34.1,nexus,gsl,glibc,OpenCASCADE-libs-modelling >= 6.3.0,OpenCASCADE-libs-foundation >= 6.3.0,muParser,numpy" )
set ( CPACK_RPM_PACKAGE_REQUIRES "boost >= 1.34.1,nexus,gsl,glibc,OCE-modeling,OCE-foundation,muParser,numpy" )
set ( CPACK_RPM_PACKAGE_REQUIRES "${CPACK_RPM_PACKAGE_REQUIRES},poco-crypto,poco-util,poco-xml,poco-net,poco-netssl,poco-foundation" )
set ( CPACK_RPM_PACKAGE_REQUIRES "${CPACK_RPM_PACKAGE_REQUIRES},gperftools-libs >= 2.0" )
set ( CPACK_RPM_PACKAGE_REQUIRES "${CPACK_RPM_PACKAGE_REQUIRES},boost-openmpi" )
Expand Down
7 changes: 7 additions & 0 deletions Code/Mantid/Build/CMake/MantidExternalData.cmake
Expand Up @@ -36,3 +36,10 @@ list(APPEND ExternalData_URL_TEMPLATES
"http://198.74.56.37/ftp/external-data/%(algo)/%(hash)"
)

# Increase network timeout defaults to avoid our slow server connection but don't override what a user provides
if(NOT ExternalData_TIMEOUT_INACTIVITY)
set(ExternalData_TIMEOUT_INACTIVITY 120)
endif()
if(NOT ExternalData_TIMEOUT_ABSOLUTE)
set(ExternalData_TIMEOUT_ABSOLUTE 600)
endif()
4 changes: 0 additions & 4 deletions Code/Mantid/Build/CMake/Packaging/launch_mantidplot.sh.in
Expand Up @@ -20,11 +20,7 @@ else
TCM_RELEASE=${TCMALLOC_RELEASE_RATE}
fi

# Define paraview information
PV_PLUGIN_PATH="${INSTALLDIR}/pvplugins/pvplugins"

# Launch
LD_PRELOAD=${LOCAL_PRELOAD} TCMALLOC_RELEASE_RATE=${TCM_RELEASE} \
LD_LIBRARY_PATH=${LOCAL_LDPATH} QT_API=pyqt \
PV_PLUGIN_PATH=${PV_PLUGIN_PATH} \
@WRAPPER_PREFIX@$INSTALLDIR/@MANTIDPLOT_EXEC@ $*@WRAPPER_POSTFIX@
2 changes: 1 addition & 1 deletion Code/Mantid/Build/CMake/ParaViewSetup.cmake
@@ -1,7 +1,7 @@
# This file will setup some common items that later setups depend on

# Set the version of ParaView that is compatible with the Mantid code base
set ( COMPATIBLE_PARAVIEW_VERSION "3.98.1" )
set ( COMPATIBLE_PARAVIEW_VERSION "4.3.1" )

# Set the name of the OSX application as this tends to be different
set ( OSX_PARAVIEW_APP "paraview.app" )
23 changes: 15 additions & 8 deletions Code/Mantid/Build/Jenkins/buildscript
Expand Up @@ -4,7 +4,7 @@
#
# Notes:
#
# WORKSPACE, JOB_NAME, NODE_LABEL GIT_COMMIT are environment variables that
# WORKSPACE, JOB_NAME, NODE_LABEL GIT_COMMIT are environment variables that
# are set by Jenkins. The last one corresponds to any labels set on a slave.
# BUILD_THREADS & PARAVIEW_DIR should be set in the configuration of each
# slave.
Expand All @@ -13,6 +13,11 @@ SCRIPT_DIR=$(dirname "$0")
# Package by default
BUILDPKG=true

###############################################################################
# All node currently have PARAVIEW_DIR=3.98.1 and PARAVIEW_NEXT_DIR=4.3.1
###############################################################################
export PARAVIEW_DIR=${PARAVIEW_NEXT_DIR}

###############################################################################
# Print out the versions of things we are using
###############################################################################
Expand Down Expand Up @@ -68,7 +73,7 @@ elif [[ $(uname) == 'Darwin' ]] ; then
if [[ ! $(command -v icpc) ]] ; then
USE_CLANG=true
fi
fi
fi

if [[ $USE_CLANG ]]; then
# Assuming we are using the clang compiler
Expand All @@ -77,14 +82,14 @@ if [[ $USE_CLANG ]]; then
export CC=clang
export CXX=clang++
#check if CMakeCache.txt exists and if so that the cxx compiler is clang++
#only needed with incremental builds. Clean builds delete this directory in a later step.
#only needed with incremental builds. Clean builds delete this directory in a later step.
if [[ -e $BUILD_DIR/CMakeCache.txt ]] && [[ ${JOB_NAME} != *clean* ]]; then
COMPILERFILEPATH=`grep 'CMAKE_CXX_COMPILER:FILEPATH' $BUILD_DIR/CMakeCache.txt`
if [[ $COMPILERFILEPATH != *clang++* ]]; then
COMPILERFILEPATH=`grep 'CMAKE_CXX_COMPILER:FILEPATH' $BUILD_DIR/CMakeCache.txt`
if [[ $COMPILERFILEPATH != *clang++* ]]; then
# Removing the build directory entirely guarantees clang is used.
rm -rf $BUILD_DIR
fi
fi
fi
fi

###############################################################################
Expand Down Expand Up @@ -119,7 +124,7 @@ fi
# RHEL6 setup steps - nodes must have a "rhel6" label set (in lowercase)
###############################################################################
if [[ ${NODE_LABELS} == *rhel6* ]]; then
SCL_ON_RHEL6="scl enable mantidlibs"
SCL_ON_RHEL6="scl enable mantidlibs34"
ON_RHEL6=true
else
SCL_ON_RHEL6="eval"
Expand All @@ -140,6 +145,8 @@ if [[ "$BUILDPKG" == true ]]; then
PACKAGINGVARS="-DENVVARS_ON_INSTALL=True -DCPACK_SET_DESTDIR=ON -DPACKAGE_DOCS=ON"
elif [[ ${JOB_NAME} == *master* ]]; then
PACKAGINGVARS="-DENVVARS_ON_INSTALL=False -DCMAKE_INSTALL_PREFIX=/opt/mantidnightly -DCPACK_PACKAGE_SUFFIX=nightly -DCPACK_SET_DESTDIR=OFF -DPACKAGE_DOCS=ON"
elif [[ ${JOB_NAME} == *pvnext* ]]; then
PACKAGINGVARS="-DENVVARS_ON_INSTALL=False -DCMAKE_INSTALL_PREFIX=/opt/mantidunstable-pvnext -DCPACK_PACKAGE_SUFFIX=unstable-pvnext -DCPACK_SET_DESTDIR=OFF"
else
PACKAGINGVARS="-DENVVARS_ON_INSTALL=False -DCMAKE_INSTALL_PREFIX=/opt/mantidunstable -DCPACK_PACKAGE_SUFFIX=unstable -DCPACK_SET_DESTDIR=OFF -DPACKAGE_DOCS=ON"
fi
Expand Down Expand Up @@ -169,7 +176,7 @@ $SCL_ON_RHEL6 "cmake -DCMAKE_BUILD_TYPE=${BUILD_CONFIG} -DENABLE_CPACK=ON -DMAKE
# Coverity build should exit early
###############################################################################
if [[ ${JOB_NAME} == *coverity_build_and_submit* ]]; then
${COVERITY_DIR}/cov-build --dir cov-int scl enable mantidlibs "make -j${BUILD_THREADS}"
${COVERITY_DIR}/cov-build --dir cov-int scl enable mantidlibs34 "make -j${BUILD_THREADS}"
tar czvf mantid.tgz cov-int
status=$(curl --form token=$COVERITY_TOKEN --form email=mantidproject@gmail.com \
--form file=@mantid.tgz --form version=$GIT_COMMIT \
Expand Down
7 changes: 6 additions & 1 deletion Code/Mantid/Build/Jenkins/buildscript.bat
Expand Up @@ -7,6 +7,12 @@ setlocal enableextensions enabledelayedexpansion
:: WORKSPACE & JOB_NAME are environment variables that are set by Jenkins.
:: BUILD_THREADS & PARAVIEW_DIR should be set in the configuration of each slave.
:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::

:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
:: All node currently have PARAVIEW_DIR=3.98.1 and PARAVIEW_NEXT_DIR=4.3.1
:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
set PARAVIEW_DIR=%PARAVIEW_NEXT_DIR%

set CMAKE_BIN_DIR=C:\Program Files (x86)\CMake 2.8\bin
"%CMAKE_BIN_DIR%\cmake.exe" --version
echo %sha1%
Expand Down Expand Up @@ -161,4 +167,3 @@ if not "%JOB_NAME%"=="%JOB_NAME:pull_requests=%" (
python !SYSTEMTESTS_DIR!\scripts\mantidinstaller.py uninstall %BUILD_DIR%
if !RETCODE! NEQ 0 exit /B 1
)

4 changes: 3 additions & 1 deletion Code/Mantid/Build/Jenkins/jenkins-slave.sh
Expand Up @@ -34,12 +34,14 @@ JAR_FILE=slave.jar
# Some versions of cron don't set the USER environment variable
# required by vnc
[ -z "$USER" ] && export USER=$(whoami)
# Put /usr/local/bin on the PATH if brew is installed
[ -f /usr/local/bin/brew ] && export PATH=${PATH}:/usr/local/bin

#####################################################################
# Script
#####################################################################
# exit if it is already running
RUNNING=$(ps u -u $(whoami) | grep java | grep ${JAR_FILE})
RUNNING=$(ps u -U $(whoami) | grep java | grep ${JAR_FILE})
if [ ! -z "${RUNNING}" ]; then
echo "Slave process is already running"
exit 0
Expand Down
98 changes: 98 additions & 0 deletions Code/Mantid/Build/Jenkins/leeroy.md
@@ -0,0 +1,98 @@
Leeroy
======

This document describes the implementation of the automated pull request Jenkins builder [leeroy](http://github.com/jfrazelle/leeroy).
Leeroy is written in Go and we are using a particular fork found [here](http://github.com/rosswhitfield/leeroy).

The user documents for the Mantid Leeroy can be found at [The automated build process](http://www.mantidproject.org/The_automated_build_process).

Setup
-----

Leeroy runs on builds.mantidproject.org on port _5000_ under the user _leeroy_, iptables were modified to allow this.

Github webhooks will need to manually be added. Add `http://builds.mantidproject.org:5000/notification/github` to be notified of pull request events.

There is a cron job to restart leeroy if it crashes (have yet to see this happen) or if the build server is restarted. There is a `start_leeroy.sh` in `/home/leeroy/` which will start leeroy with the correct arguments, which the cron job calls.

The config file for leeroy can be found at /home/leeroy/config.json and is copied at the bottom of this document with jenkins and github tokens removed. Leeroy uses the _mantid-builder_ account to access Jenkins and Github via API tokens.

Adding a new job to Leeroy
--------------------------

To setup a new Jenkins jobs to work with Leeroy there are three additional thing needed over the standard settings for a job in Jenkins.

* Add a Job Notifications with `URL` set to `http://localhost:5000/notification/jenkins` and `Log` to `1`.
* Add 5 string parameters; `GIT_BASE_REPO`, `GIT_HEAD_REPO`, `GIT_SHA1`, `GITHUB_URL` and `PR`.
* Under Source Code Management set Repository `URL` to `git://github.com/$GIT_HEAD_REPO.git`, Refspec (under Advanced) to `+refs/pull/$PR/merge:pull-$PR-merged` and Branch Specifier to `pull-$PR-merged`.
* Add the build to `config.json` and restart Leeroy.


Other things
------------

Using the `Refspec` in the job config will mean that the pull request branch will be merged into the target before it is built. If it can't be merged cleanly the pull request will fail.

Leeroy can be manually triggered outside of the Jenkins Rebuild by running for example
```shell
curl -u USER:PASS builds.mantidproject.org:5000/build/retry -d '{"context":"cppcheck","repo":"mantidproject/mantid","number":415}'
```
which will start the cppcheck build on pull request 415

A cron jobs runs `/home/leeroy/cron_leeroy.sh` every 10 minutes to pick up missed builds, which executes the following type command.
```shell
curl -u USER:PASS builds.mantidproject.org:5000/build/cron -d '{"context":"cppcheck","repo":"mantidproject/mantid"}'
```

Config file (`config.json`)
-----------
```json
{
"jenkins": {
"username": "mantid-builder",
"token": "",
"base_url": "http://builds.mantidproject.org"
},
"build_commits": "last",
"github_token": "",
"builds": [
{
"github_repo": "mantidproject/mantid",
"jenkins_job_name": "pull_requests-rhel7",
"context": "RHEL7"
},
{
"github_repo": "mantidproject/mantid",
"jenkins_job_name": "pull_requests-rhel6",
"context": "RHEL6"
},
{
"github_repo": "mantidproject/mantid",
"jenkins_job_name": "pull_requests-ubuntu",
"context": "Ubuntu"
},
{
"github_repo": "mantidproject/mantid",
"jenkins_job_name": "pull_requests-win7",
"context": "Windows"
},
{
"github_repo": "mantidproject/mantid",
"jenkins_job_name": "pull_requests-osx",
"context": "OSX"
},
{
"github_repo": "mantidproject/mantid",
"jenkins_job_name": "pull_requests-cppcheck",
"context": "cppcheck"
},
{
"github_repo": "mantidproject/mantid",
"jenkins_job_name": "pull_requests-doxygen",
"context": "Doxygen"
}
],
"user": "USER",
"pass": "PASS"
}
```
2 changes: 1 addition & 1 deletion Code/Mantid/Build/Jenkins/systemtests
Expand Up @@ -28,7 +28,7 @@ fi
# RHEL6 setup steps - nodes must have a "rhel6" label set (in lowercase)
###############################################################################
if [[ ${NODE_LABELS} == *rhel6* ]]; then
SCL_ON_RHEL6="scl enable mantidlibs"
SCL_ON_RHEL6="scl enable mantidlibs34"
ON_RHEL6=true
else
SCL_ON_RHEL6="eval"
Expand Down
Expand Up @@ -57,8 +57,8 @@ Requires: texlive-was
Requires: tex-preview
Requires: dvipng
%if 0%{?el6}
Requires: mantidlibs-qt-devel
Requires: mantidlibs-qtwebkit-devel
Requires: mantidlibs34-qt-devel
Requires: mantidlibs34-qtwebkit-devel
Requires: scl-utils
%else
Requires: qt-devel
Expand Down

0 comments on commit f93593a

Please sign in to comment.