Re #9853 Merge branch 'master' into

feature/9843_integrate_sysmonitor_widget

Conflicts:
	Code/Mantid/MantidPlot/CMakeLists.txt
	Code/Mantid/MantidPlot/src/ApplicationWindow.cpp

CONTRIBUTING.md

@@ -1,14 +1,13 @@
-The [developer documentation](http://www.mantidproject.org/Category:Development) has information on how to participate in the mantid project as a developer. This document is meant to outline the steps for contributing to mantid without becomming a developer.
+The [developer documentation](http://www.mantidproject.org/Category:Development) has information on how to participate in the mantid project as a developer. This document is meant to outline the steps for contributing to mantid without becomming a developer. We aspire to have similar guidelines as [github](https://github.com/blog/1943-how-to-write-the-perfect-pull-request).
 
  1. [Fork](https://help.github.com/articles/fork-a-repo) the repository.
  2. Make changes as you see fit. Please still follow the guidelines for [running the unit tests](http://www.mantidproject.org/Running_the_unit_tests) and the [build servers](http://www.mantidproject.org/The_automated_build_process).
- 3. Before submitting pull requests email mantid-help@mantidproject.org, and a developer will set up a branch that you can submit pull requests to.
- 4. Submit a [pull request](https://help.github.com/articles/using-pull-requests) to this branch.
+ 4. Submit a [pull request](https://help.github.com/articles/using-pull-requests) to this branch. This is a start to the conversation.
+ 3. (Optional) email mantid-help@mantidproject.org requests to.
 
-
-Hints to make the integration of your changes easy:
+Hints to make the integration of your changes easy (and happen faster):
 - Keep your pull requests small
+- Some comments are part of the build server infrastructure (e.g. 'test this please')
 - Don't forget your unit tests
 - All algorithms need documentation, don't forget the .rst file
-- Contact us before creating the pull request
 - Don't take changes requests to change your code personally

Code/Mantid/Build/CMake/CommonSetup.cmake

@@ -54,7 +54,7 @@ add_definitions ( -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG )
 find_package ( Poco 1.4.2 REQUIRED )
 include_directories( SYSTEM ${POCO_INCLUDE_DIRS} )
 
-find_package ( Nexus 4.3.0 REQUIRED )
+find_package ( Nexus 4.3.1 REQUIRED )
 include_directories ( SYSTEM ${NEXUS_INCLUDE_DIR} )
 
 find_package ( MuParser REQUIRED )
@@ -301,6 +301,35 @@ else()
   message ( STATUS "Could not find Squish - GUI testing not available. Try specifying your SQUISH_INSTALL_DIR cmake variable." )
 endif()
 
+###########################################################################
+# External Data for testing
+###########################################################################
+if ( CXXTEST_FOUND OR PYUNITTEST_FOUND )
+ include ( MantidExternalData )
+
+# None of our tests reference files directly as arguments so we have to manually
+# call ExternalData_Expand_Arguments to register the files with the ExternalData
+# mechanism
+get_filename_component ( EXTERNALDATATEST_SOURCE_DIR ${PROJECT_SOURCE_DIR} ABSOLUTE )
+file( GLOB_RECURSE doctest_content_links
+  RELATIVE "${EXTERNALDATATEST_SOURCE_DIR}" "Testing/Data/DocTest/*.md5" )
+file( GLOB_RECURSE unittest_content_links
+  RELATIVE "${EXTERNALDATATEST_SOURCE_DIR}" "Testing/Data/UnitTest/*.md5" )
+set ( content_links "${doctest_content_links};${unittest_content_links}" )
+foreach(link ${content_links})
+  string( REGEX REPLACE "\\.md5$" "" link ${link} )
+  ExternalData_Expand_Arguments( StandardTestData
+    link_location
+    DATA{${link}}
+    )
+endforeach()
+
+# Create target to download data from the StandardTestData group.  This must come after
+# all tests have been added that reference the group, so we put it last.
+ExternalData_Add_Target(StandardTestData)
+set_target_properties(StandardTestData PROPERTIES EXCLUDE_FROM_ALL TRUE)
+endif()
+
 ###########################################################################
 # Set a flag to indicate that this script has been called
 ###########################################################################

Code/Mantid/Build/CMake/DarwinSetup.cmake

@@ -102,6 +102,7 @@ endif()
 ###########################################################################
 # Mac-specific installation setup
 ###########################################################################
+
 set ( CMAKE_INSTALL_PREFIX "" )
 set ( CPACK_PACKAGE_EXECUTABLES MantidPlot )
 set ( INBUNDLE MantidPlot.app/ )
@@ -119,27 +120,50 @@ set ( PVPLUGINS_DIR MantidPlot.app/pvplugins )
 set ( PVPLUGINS_SUBDIR pvplugins ) # Need to tidy these things up!
 
 if (OSX_VERSION VERSION_LESS 10.9)
+ set ( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/${LIB_DIR};${CMAKE_INSTALL_PREFIX}/${PLUGINS_DIR};${CMAKE_INSTALL_PREFIX}/${PVPLUGINS_DIR} )
  set ( PYQT4_PYTHONPATH /Library/Python/${PY_VER}/site-packages/PyQt4 )
  set ( SITEPACKAGES /Library/Python/${PY_VER}/site-packages )
 else()
+ set(CMAKE_MACOSX_RPATH 1)
  # Assume we are using homebrew for now
  # set Deployment target to 10.8
  set ( CMAKE_OSX_SYSROOT /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX10.9.sdk )
  set ( CMAKE_OSX_ARCHITECTURES x86_64 )
  set ( CMAKE_OSX_DEPLOYMENT_TARGET 10.8 )
- set ( PYQT4_PYTHONPATH /usr/local/lib/python${PY_VER}/site-packages/PyQt4 )
- set ( SITEPACKAGES /usr/local/lib/python${PY_VER}/site-packages )
+ # Follow symlinks so cmake copies the file
+ # PYQT4_PATH, SITEPACKAGES_PATH, OPENSSL_ROOT_DIR may be defined externally (cmake -D)
+ # it would be good do not overwrite them (important for the compilation with macports)
+ if (NOT PYQT4_PATH)
+  set ( PYQT4_PATH /usr/local/lib/python${PY_VER}/site-packages/PyQt4 )
+ endif(NOT PYQT4_PATH)
+ execute_process(COMMAND readlink ${PYQT4_PATH}/Qt.so OUTPUT_VARIABLE PYQT4_SYMLINK_Qtso)
+ string(FIND "${PYQT4_SYMLINK_Qtso}" "Qt.so" STOPPOS)
+ string(SUBSTRING "${PYQT4_SYMLINK_Qtso}" 0 ${STOPPOS} PYQT4_SYMLINK)
+ set  ( PYQT4_PYTHONPATH ${PYQT4_PATH}/${PYQT4_SYMLINK} )
+ string(REGEX REPLACE "/$" "" PYQT4_PYTHONPATH "${PYQT4_PYTHONPATH}")
+
+ if (NOT SITEPACKAGES_PATH)
+   set ( SITEPACKAGES_PATH /usr/local/lib/python${PY_VER}/site-packages )
+ endif(NOT SITEPACKAGES_PATH)
+ execute_process(COMMAND readlink ${SITEPACKAGES_PATH}/sip.so OUTPUT_VARIABLE SITEPACKAGES_SYMLINK_sipso)
+ string(FIND "${SITEPACKAGES_SYMLINK_sipso}" "sip.so" STOPPOS)
+ string(SUBSTRING "${SITEPACKAGES_SYMLINK_sipso}" 0 ${STOPPOS} SITEPACKAGES_SYMLINK)
+ set  ( SITEPACKAGES ${SITEPACKAGES_PATH}/${SITEPACKAGES_SYMLINK} )
+ string(REGEX REPLACE "/$" "" SITEPACKAGES "${SITEPACKAGES}")
+
  # use homebrew OpenSSL package
- set ( OPENSSL_ROOT_DIR /usr/local/opt/openssl )
+ if (NOT OPENSSL_ROOT_DIR)
+   set ( OPENSSL_ROOT_DIR /usr/local/opt/openssl )
+ endif(NOT OPENSSL_ROOT_DIR)
 endif()
 
 # Python packages
 
 install ( PROGRAMS ${SITEPACKAGES}/sip.so DESTINATION ${BIN_DIR} )
 
 # Explicitly specify which PyQt libraries we want because just taking the whole
-# directory will swell the install kit unnecessarily.
-install ( FILES ${PYQT4_PYTHONPATH}/Qt.so
+#directory will swell the install kit unnecessarily.
+ install ( FILES ${PYQT4_PYTHONPATH}/Qt.so
                 ${PYQT4_PYTHONPATH}/QtCore.so
                 ${PYQT4_PYTHONPATH}/QtGui.so
                 ${PYQT4_PYTHONPATH}/QtOpenGL.so
@@ -156,16 +180,19 @@ endif ()
 
 install ( DIRECTORY ${PYQT4_PYTHONPATH}/uic DESTINATION ${BIN_DIR}/PyQt4 )
 
-# Python packages in Third_Party need copying to build directory and the final package
-file ( GLOB THIRDPARTY_PYTHON_PACKAGES ${CMAKE_LIBRARY_PATH}/Python/* )
-foreach ( PYPACKAGE ${THIRDPARTY_PYTHON_PACKAGES} )
-  if ( IS_DIRECTORY ${PYPACKAGE} )
-    install ( DIRECTORY ${PYPACKAGE} DESTINATION ${BIN_DIR} USE_SOURCE_PERMISSIONS )
-  else()
-    install ( FILES ${PYPACKAGE} DESTINATION ${BIN_DIR} )
-  endif()
-  file ( COPY ${PYPACKAGE} DESTINATION ${PROJECT_BINARY_DIR}/bin )
-endforeach( PYPACKAGE )
+# done as part of packaging step in 10.9+ builds.
+if (OSX_VERSION VERSION_LESS 10.9)
+  # Python packages in Third_Party need copying to build directory and the final package
+  file ( GLOB THIRDPARTY_PYTHON_PACKAGES ${CMAKE_LIBRARY_PATH}/Python/* )
+  foreach ( PYPACKAGE ${THIRDPARTY_PYTHON_PACKAGES} )
+    if ( IS_DIRECTORY ${PYPACKAGE} )
+      install ( DIRECTORY ${PYPACKAGE} DESTINATION ${BIN_DIR} USE_SOURCE_PERMISSIONS )
+    else()
+      install ( FILES ${PYPACKAGE} DESTINATION ${BIN_DIR} )
+    endif()
+    file ( COPY ${PYPACKAGE} DESTINATION ${PROJECT_BINARY_DIR}/bin )
+  endforeach( PYPACKAGE )
+endif ()
 
 install ( DIRECTORY ${QT_PLUGINS_DIR}/imageformats DESTINATION MantidPlot.app/Contents/Frameworks/plugins )
 install ( DIRECTORY ${QT_PLUGINS_DIR}/sqldrivers DESTINATION MantidPlot.app/Contents/Frameworks/plugins )

Code/Mantid/Build/CMake/LinuxPackageScripts.cmake

@@ -23,9 +23,19 @@ endif()
 set ( CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_PREFIX}/${LIB_DIR};${CMAKE_INSTALL_PREFIX}/${PLUGINS_DIR};${CMAKE_INSTALL_PREFIX}/${PVPLUGINS_DIR} )
 
 ###########################################################################
-# LD_PRELOAD libraries
+# LD_PRELOAD TCMalloc
 ###########################################################################
-set ( EXTRA_LDPRELOAD_LIBS "${TCMALLOC_LIBRARIES}" )
+# User systems will only have the libraries and not the symbolic link
+# so we must set the preload to the versioned library. We will assume it is
+# in the same location as the system that the package was built on
+if ( TCMALLOC_LIBRARIES )
+  execute_process ( COMMAND readlink --no-newline --canonicalize-existing ${TCMALLOC_LIBRARIES}
+                    OUTPUT_VARIABLE TCMALLOC_PRELOAD
+                    RESULT_VARIABLE READLINK_RESULT )
+  if ( READLINK_RESULT EQUAL 1 )
+    message ( FATAL_ERROR "Unable to find real file that tcmalloc symlink, ${TCMALLOC_LIBRARIES}, points to." )
+  endif()
+endif()
 
 ###########################################################################
 # Environment scripts (profile.d)
@@ -84,10 +94,12 @@ set ( POST_UNINSTALL_FILE ${CMAKE_CURRENT_BINARY_DIR}/postrm )
 
 if ( "${UNIX_DIST}" MATCHES "RedHatEnterprise" OR "${UNIX_DIST}" MATCHES "^Fedora" ) # RHEL/Fedora
   if ( "${UNIX_CODENAME}" MATCHES "Santiago" ) # el6
-    set ( WRAPPER_COMMAND "scl enable mantidlibs" )
+    set ( WRAPPER_PREFIX "scl enable mantidlibs \"" )
+    set ( WRAPPER_POSTFIX "\"" )
     set ( EXTRA_LDPATH "/usr/lib64/paraview" )
   else()
-    set ( WRAPPER_COMMAND "eval" )
+    set ( WRAPPER_PREFIX "" )
+    set ( WRAPPER_POSTFIX "" )
   endif()
 
   if ( NOT MPI_BUILD )
@@ -106,7 +118,8 @@ if ( "${UNIX_DIST}" MATCHES "RedHatEnterprise" OR "${UNIX_DIST}" MATCHES "^Fedor
     set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE ${POST_UNINSTALL_FILE} )
   endif()
 elseif ( "${UNIX_DIST}" MATCHES "Ubuntu" )
-  set ( WRAPPER_COMMAND "eval" )
+  set ( WRAPPER_PREFIX "" )
+  set ( WRAPPER_POSTFIX "" )
 
   if ( NOT MPI_BUILD )
     configure_file ( ${CMAKE_MODULE_PATH}/Packaging/deb/scripts/deb_pre_inst.in

Code/Mantid/Build/CMake/MantidExternalData.cmake

@@ -0,0 +1,38 @@
+################################################################################
+# Setup the ExternalData variables for the project
+################################################################################
+include(ExternalData)
+
+if(NOT MANTID_DATA_STORE)
+  # Select a default in the home directory
+  set(MANTID_DATA_STORE_DEFAULT "$ENV{HOME}/MantidExternalData")
+endif()
+
+# Provide users with an option to select a local object store,
+# starting with the above-selected default.
+set(MANTID_DATA_STORE "${MANTID_DATA_STORE_DEFAULT}" CACHE PATH
+  "Local directory holding ExternalData objects in the layout %(algo)/%(hash).")
+mark_as_advanced(MANTID_DATA_STORE)
+
+if(NOT MANTID_DATA_STORE)
+  message(FATAL_ERROR "MANTID_DATA_STORE not set. It is required for external data")
+endif()
+
+# Tell ExternalData module about selected object stores.
+list(APPEND ExternalData_OBJECT_STORES
+  # Store selected by Mantid-specific configuration above.
+  ${MANTID_DATA_STORE}
+)
+
+# Default binary root to build directory
+set(ExternalData_BINARY_ROOT ${CMAKE_BINARY_DIR}/ExternalData CACHE STRING
+  "A directory holding the links (copies on windows) to the real content files.")
+
+set(ExternalData_URL_TEMPLATES "" CACHE STRING
+  "Additional URL templates for the ExternalData CMake script to look for testing data. E.g.
+file:///var/bigharddrive/%(algo)/%(hash)")
+mark_as_advanced(ExternalData_URL_TEMPLATES)
+list(APPEND ExternalData_URL_TEMPLATES
+  "http://198.74.56.37/ftp/external-data/%(algo)/%(hash)"
+)
+

Code/Mantid/Build/CMake/Packaging/launch_mantidplot.sh.in

@@ -9,7 +9,7 @@ SCRIPTFILE=$(readlink -f "$0")
 INSTALLDIR=$(echo $SCRIPTFILE | sed -r -e 's|^(.*)/(.*)$|\1|g') #.* is greedy and eats up until the final slash
 
 # Define extra libraries and load paths
-LOCAL_PRELOAD=$(readlink -f @EXTRA_LDPRELOAD_LIBS@)
+LOCAL_PRELOAD=@TCMALLOC_PRELOAD@
 if [ -n "${LD_PRELOAD}" ]; then
     LOCAL_PRELOAD=${LOCAL_PRELOAD}:${LD_PRELOAD}
 fi
@@ -21,4 +21,4 @@ else
 fi
 
 # Launch
-LD_PRELOAD=${LOCAL_PRELOAD} TCMALLOC_RELEASE_RATE=${TCM_RELEASE} LD_LIBRARY_PATH=${LOCAL_LDPATH} @WRAPPER_COMMAND@ "$INSTALLDIR/@MANTIDPLOT_EXEC@ $*" 
+LD_PRELOAD=${LOCAL_PRELOAD} TCMALLOC_RELEASE_RATE=${TCM_RELEASE} LD_LIBRARY_PATH=${LOCAL_LDPATH} @WRAPPER_PREFIX@$INSTALLDIR/@MANTIDPLOT_EXEC@ $*@WRAPPER_POSTFIX@

Code/Mantid/Build/CMake/VersionNumber.cmake

@@ -1,7 +1,7 @@
 # Set the version number here for MantidVersion and the package filenames
 
 set ( VERSION_MAJOR 3 )
-set ( VERSION_MINOR 2 )
+set ( VERSION_MINOR 3 )
 
 # UNCOMMENT the next 'set' line to 'force' the patch version number to
 # a value (instead of using the count coming out of 'git describe')

Code/Mantid/Build/Jenkins/buildscript

@@ -15,28 +15,49 @@
 cmake --version
 echo "SHA1=${sha1}"
 
+if [[ ${JOB_NAME} == *clang* ]]; then
+  # Assuming we are using the clang compiler
+  echo "Using clang/llvm compiler."
+  clang --version
+  export CC=clang
+  export CXX=clang++
+  #check if CMakeCache.txt exists and if so that the cxx compiler is clang++
+  #only needed with incremental builds. Clean builds delete this directory in a later step. 
+  if [[ -e $WORKSPACE/build/CMakeCache.txt ]] && [[ ${JOB_NAME} != *clean* ]]; then
+    COMPILERFILEPATH=`grep 'CMAKE_CXX_COMPILER:FILEPATH' $WORKSPACE/build/CMakeCache.txt` 
+    if [[ $COMPILERFILEPATH != *clang++* ]]; then 
+      # Removing the build directory entirely guarantees clang is used.
+      rm -rf $WORKSPACE/build
+    fi
+  fi 
+fi
+
+###############################################################################
+# OS X 10.8 setup steps
+###############################################################################
+if [[ $(uname) == 'Darwin' ]] && [[ ${JOB_NAME} != *clang* ]]; then
+  # Assuming we are using the Intel compiler
+  cd $WORKSPACE/Code
+  ./fetch_Third_Party.sh
+  cd $WORKSPACE
+  # Setup environment variables
+  source /opt/intel/bin/iccvars.sh intel64
+  export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$WORKSPACE/Code/Third_Party/lib/mac64:/Library/Frameworks
+  # Make sure we pick up the Intel compiler
+  export CC=icc
+  export CXX=icpc
+  echo "Using Intel compiler."
+  icpc --version
+fi
+
 ###############################################################################
-# OS X setup steps
+# Set up the location for the local object store outside of the build and
+# source tree, which can be shared by multiple builds.
+# It defaults to the parent directory of the workspace but can be overridden
+# by setting the MANTID_DATA_STORE environment variable.
 ###############################################################################
-if [[ $(uname) == 'Darwin' ]]; then
-  if [[ ${JOB_NAME} == *clang* ]]; then
-    # Assuming we are using the clang compiler
-    echo "Using clang/llvm compiler."
-    clang --version
-  else
-    # Assuming we are using the Intel compiler
-    cd $WORKSPACE/Code
-    ./fetch_Third_Party.sh
-    cd $WORKSPACE
-    # Setup environment variables
-    source /opt/intel/bin/iccvars.sh intel64
-    export DYLD_LIBRARY_PATH=$DYLD_LIBRARY_PATH:$WORKSPACE/Code/Third_Party/lib/mac64:/Library/Frameworks
-    # Make sure we pick up the Intel compiler
-    export CC=icc
-    export CXX=icpc
-    echo "Using Intel compiler."
-    icpc --version
-  fi
+if [ -z "$MANTID_DATA_STORE" ]; then
+  export MANTID_DATA_STORE=$(dirname $WORKSPACE)
 fi
 
 ###############################################################################
@@ -93,7 +114,7 @@ fi
 ###############################################################################
 # CMake configuration
 ###############################################################################
-$SCL_ON_RHEL6 "cmake -DCMAKE_BUILD_TYPE=${BUILD_CONFIG} -DENABLE_CPACK=ON -DMAKE_VATES=ON -DParaView_DIR=${PARAVIEW_DIR} -DDOCS_HTML=ON ${PACKAGINGVARS} ../Code/Mantid"
+$SCL_ON_RHEL6 "cmake -DCMAKE_BUILD_TYPE=${BUILD_CONFIG} -DENABLE_CPACK=ON -DMAKE_VATES=ON -DParaView_DIR=${PARAVIEW_DIR} -DMANTID_DATA_STORE=${MANTID_DATA_STORE} -DDOCS_HTML=ON ${PACKAGINGVARS} ../Code/Mantid"
 
 ###############################################################################
 # Coverity build should exit early

Code/Mantid/Build/Jenkins/buildscript.bat

@@ -32,6 +32,16 @@ cd %WORKSPACE%
 
 set PATH=%WORKSPACE%\Code\Third_Party\lib\win64;%WORKSPACE%\Code\Third_Party\lib\win64\Python27;%PARAVIEW_DIR%\bin\%BUILD_CONFIG%;%PATH%
 
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+:: Set up the location for local object store outside of the build and source
+:: tree, which can be shared by multiple builds.
+:: It defaults to the parent directory of the workspace but can be overridden
+:: by setting the MANTID_DATA_STORE environment variable.
+:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
+if NOT DEFINED MANTID_DATA_STORE (
+  for %%F in ("%WORKSPACE%") do set MANTID_DATA_STORE=%%~dpF
+)
+
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 :: Check whether this is a clean build (must have 'clean' in the job name)
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
@@ -53,7 +63,7 @@ cd %WORKSPACE%\build
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
 :: CMake configuration
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::
-"C:\Program Files (x86)\CMake 2.8\bin\cmake.exe" -G "Visual Studio 11 Win64" -DCONSOLE=OFF -DENABLE_CPACK=ON -DMAKE_VATES=ON -DParaView_DIR=%PARAVIEW_DIR% -DUSE_PRECOMPILED_HEADERS=ON %PACKAGE_DOCS% ..\Code\Mantid
+"C:\Program Files (x86)\CMake 2.8\bin\cmake.exe" -G "Visual Studio 11 Win64" -DCONSOLE=OFF -DENABLE_CPACK=ON -DMAKE_VATES=ON -DParaView_DIR=%PARAVIEW_DIR% -DMANTID_DATA_STORE=%MANTID_DATA_STORE% -DUSE_PRECOMPILED_HEADERS=ON %PACKAGE_DOCS% ..\Code\Mantid
 if ERRORLEVEL 1 exit /B %ERRORLEVEL%
 
 :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::