Skip to content

Commit

Permalink
Merge branch 'multithreading' into 'master'
Browse files Browse the repository at this point in the history
Implement Event-based Multithreading

Closes #8, #4, #5, #7, #184, #203, #166, and #107

See merge request allpix-squared/allpix-squared!231
  • Loading branch information
simonspa committed Apr 6, 2021
2 parents 8e36b51 + 4a97f70 commit acb8629
Show file tree
Hide file tree
Showing 230 changed files with 5,525 additions and 2,615 deletions.
35 changes: 21 additions & 14 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ stages:
script:
- mkdir build
- cd build
- export CCACHE_DIR=`pwd`/ccache
- cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_LCIOWriter=ON -DCMAKE_BUILD_TYPE=RELEASE -DROOT_DIR=$ROOTSYS -DEigen3_DIR=$Eigen3_DIR -DLCIO_DIR=$LCIO_DIR ..
- ninja -k0
- ninja install
Expand Down Expand Up @@ -58,7 +59,7 @@ cmp:lxplus-gcc:
cmp:cc7-docker:
extends: .compile
image:
name: gitlab-registry.cern.ch/allpix-squared/allpix-squared/allpix-squared-deps:latest
name: gitlab-registry.cern.ch/allpix-squared/allpix-squared/allpix-squared-deps-mt:latest
entrypoint: [""]
before_script:
- source scl_source enable devtoolset-8 || echo " "
Expand Down Expand Up @@ -106,6 +107,7 @@ cmp:mac1015-clang:
stage: formatting
tags:
- docker
image: gitlab-registry.cern.ch/sft/docker/centos7:latest
before_script:
- export COMPILER_TYPE="llvm"
- source .ci/init_x86_64.sh
Expand All @@ -114,7 +116,6 @@ fmt:cc7-llvm-format:
extends: .format
needs: []
dependencies: []
image: gitlab-registry.cern.ch/sft/docker/centos7:latest
script:
- mkdir -p build
- cd build/
Expand All @@ -125,7 +126,6 @@ fmt:cc7-llvm-lint:
extends: .format
needs: []
dependencies: []
image: gitlab-registry.cern.ch/sft/docker/centos7:latest
script:
- mkdir -p build
- cd build/
Expand All @@ -135,7 +135,6 @@ fmt:cc7-llvm-lint:

fmt:coverage:
extends: .format
image: gitlab-registry.cern.ch/sft/docker/centos7:latest
needs: []
dependencies: []
before_script:
Expand All @@ -146,14 +145,14 @@ fmt:coverage:
- pip install --trusted-host=pypi.org --user gcovr
- mkdir build
- cd build
- export CCACHE_DIR=`pwd`/ccache
- cmake -GNinja -DCMAKE_CXX_FLAGS="-Werror" -DBUILD_LCIOWriter=ON -DCMAKE_BUILD_TYPE=Debug -DCOVERAGE=ON -DLCIO_DIR=$LCIO_DIR ..
- ninja -k0
- ninja install
- ninja coverage_gcovr

fmt:codespell:
extends: .format
image: gitlab-registry.cern.ch/sft/docker/centos7:latest
needs: []
dependencies: []
script:
Expand All @@ -163,7 +162,6 @@ fmt:codespell:

fmt:coverity:
extends: .format
image: gitlab-registry.cern.ch/sft/docker/centos7:latest
needs: []
dependencies: []
only:
Expand All @@ -189,18 +187,29 @@ fmt:coverity:
stage: testing
tags:
- docker
after_script:
- source .ci/init_x86_64.sh
- ./.gitlab/ci/transform_ctest_junit.py build/Testing/`head -n 1 build/Testing/TAG`/Test.xml .gitlab/ci/ctest-to-junit.xsl apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml
artifacts:
when: always
expire_in: 1 week
name: "apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}"
paths:
- apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml
reports:
junit: apsq-${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}.xml

.testmod:
extends: .test
script:
- cd build/
- ctest -R test_modules --output-on-failure -j4
- ctest -R test_modules --no-compress-output --test-action Test -j4

.testcore:
extends: .test
script:
- cd build/
- ctest -R test_core --output-on-failure -j4
- ctest -R test_core --no-compress-output --test-action Test -j4


# Test if examples still execute fine:
Expand All @@ -215,7 +224,7 @@ examples:
- source .ci/init_x86_64.sh
script:
- cd build/
- ctest -R examples --output-on-failure -j4
- ctest -R examples --no-compress-output --test-action Test -j4

# SLC 6

Expand Down Expand Up @@ -273,7 +282,7 @@ core:cc7-docker:
- job: cmp:cc7-docker
artifacts: true
image:
name: gitlab-registry.cern.ch/allpix-squared/allpix-squared/allpix-squared-deps:latest
name: gitlab-registry.cern.ch/allpix-squared/allpix-squared/allpix-squared-deps-mt:latest
entrypoint: [""]
before_script:
- source scl_source enable devtoolset-8 || echo " "
Expand Down Expand Up @@ -396,6 +405,8 @@ perf:cc7-gcc:

.doc:
stage: documentation
needs: []
dependencies: []
tags:
- docker
artifacts:
Expand All @@ -408,8 +419,6 @@ perf:cc7-gcc:
cmp:doxygen:
extends: .doc
image: gitlab-registry.cern.ch/sft/docker/centos7:latest
needs: []
dependencies: []
script:
- source .ci/init_x86_64.sh
- mkdir -p public/usermanual
Expand All @@ -423,8 +432,6 @@ cmp:doxygen:
cmp:usermanual:
extends: .doc
image: gitlab-registry.cern.ch/clicdp/publications/templates/custom_ci_worker:fedora-latex-latest
needs: []
dependencies: []
script:
- mkdir -p public/usermanual
- mkdir build
Expand Down
36 changes: 36 additions & 0 deletions .gitlab/ci/ctest-to-junit.xsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="xml" indent="yes" />
<xsl:template match="/">
<testsuites>
<xsl:variable name="buildName" select="//Site/@BuildName"/>
<xsl:variable name="numberOfTests" select="count(//Site/Testing/Test)"/>
<xsl:variable name="numberOfFailures" select="count(//Site/Testing/Test[@Status!='passed'])" />
<testsuite name="CTest"
tests="{$numberOfTests}" time="0"
failures="{$numberOfFailures}" errors="0"
skipped="0">
<xsl:for-each select="//Site/Testing/Test">
<xsl:variable name="testName" select="translate(Name, '-', '_')"/>
<xsl:variable name="duration" select="Results/NamedMeasurement[@name='Execution Time']/Value"/>
<xsl:variable name="status" select="@Status"/>
<xsl:variable name="output" select="Results/Measurement/Value"/>
<xsl:variable name="className" select="translate(Path, '/.', '.')"/>
<testcase classname="projectroot{$className}"
name="{$testName}"
time="{$duration}">
<xsl:if test="@Status!='passed'">
<failure>
<xsl:value-of select="$output" />
</failure>
</xsl:if>
<system-out>
<xsl:value-of select="$output" />
</system-out>
</testcase>
</xsl:for-each>
</testsuite>
</testsuites>
</xsl:template>
</xsl:stylesheet>

25 changes: 25 additions & 0 deletions .gitlab/ci/transform_ctest_junit.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#!/usr/bin/env python

import sys
import lxml.etree as et

# Check if we got token:
if len(sys.argv) != 4:
sys.exit(1)

input_xml = sys.argv[1]
input_xsl = sys.argv[2]
output_xml = sys.argv[3]

print("Reading XML input file")
dom = et.parse(input_xml)

print("Reading XSLT file")
xslt = et.parse(input_xsl)

print("Transforming DOM")
transform = et.XSLT(xslt)
newdom = transform(dom)

print("Writing XML output file")
newdom.write(output_xml, pretty_print=True)
18 changes: 18 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,13 @@ SET(COMPILER_FLAGS -pedantic -Wall -Wextra -Wcast-align -Wcast-qual -Wconversion

INCLUDE("cmake/compiler-flag-checks.cmake")

# Use CCache if available
FIND_PROGRAM(CCACHE_FOUND ccache)
IF(CCACHE_FOUND)
MESSAGE(STATUS "Using Ccache: ${CCACHE_FOUND}")
SET(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_FOUND}" CACHE PATH "CCache program" FORCE)
ENDIF()

# Check for C++17 and require C++17 if available else fallback to C++14 (should currently be sufficient)
CHECK_CXX_COMPILER_FLAG(-std=c++17 SUPPORT_STD_CXX17)
IF(SUPPORT_STD_CXX17)
Expand Down Expand Up @@ -160,6 +167,10 @@ FIND_PACKAGE(Threads REQUIRED)
# Define the libraries
SET(ALLPIX_LIBRARIES "")

# Define the optional libraries
SET(ALLPIX_GEANT4_INTERFACE "AllpixGeant4Interface")
SET(ALLPIX_BUILD_GEANT4_INTERFACE "OFF" CACHE BOOL "Build Geant4 interface library" FORCE)

# ROOT is required for vector and persistency etc
FIND_PACKAGE(ROOT REQUIRED COMPONENTS Geom Core GenVector Hist RIO NO_MODULE)
IF(NOT ROOT_FOUND)
Expand Down Expand Up @@ -206,6 +217,8 @@ ADD_RUNTIME_DEP(thisroot.sh)

# Set the clang-format version required by the CI for correct formatting:
SET(CLANG_FORMAT_VERSION "10")
# Set the clang-tidy version of the linter required by the CI:
SET(CLANG_TIDY_VERSION "10")

# Set the source files to clang-format (FIXME: determine this better)
FILE(GLOB_RECURSE
Expand Down Expand Up @@ -240,6 +253,11 @@ SET(ALLPIX_LIBRARIES ${ALLPIX_LIBRARIES} AllpixCore)
# Build required modules
ADD_SUBDIRECTORY(src/modules)

# Build geant4 interface library if needed by modules
IF(${ALLPIX_BUILD_GEANT4_INTERFACE})
ADD_SUBDIRECTORY(src/tools/geant4)
ENDIF()

# Build the executable
ADD_SUBDIRECTORY(src/exec)

Expand Down
40 changes: 25 additions & 15 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,22 @@
# Allpix<sup>2</sup>
### Generic Pixel Detector Simulation Framework

Allpix<sup>2</sup> is a generic simulation framework for silicon detectors, written in modern C++. The goal of the Allpix<sup>2</sup> framework is to provide a comprehensive and easy-to-use package for simulating the performance of silicon detectors from incident ionizing radiation to the digitization of pixel hits in the detector ASIC.

For more details about the project please have a look at the website at https://cern.ch/allpix-squared.

[![build status](https://gitlab.cern.ch/allpix-squared/allpix-squared/badges/master/build.svg)](https://gitlab.cern.ch/allpix-squared/allpix-squared/commits/master)
[![coverity status](https://scan.coverity.com/projects/11975/badge.svg)](https://scan.coverity.com/projects/koensw-allpix-squared)
[![build status](https://gitlab.cern.ch/allpix-squared/allpix-squared/badges/master/pipeline.svg)](https://gitlab.cern.ch/allpix-squared/allpix-squared/commits/master)
[![coverity status](https://scan.coverity.com/projects/21520/badge.svg)](https://scan.coverity.com/projects/allpix-squared)
[![zenodo record](https://zenodo.org/badge/DOI/10.5281/zenodo.3550935.svg)](https://doi.org/10.5281/zenodo.3550935)


## Dependencies
* [ROOT](https://root.cern.ch/building-root) (required, with the GenVector component)
* [Geant4](http://geant4-userdoc.web.cern.ch/geant4-userdoc/UsersGuides/InstallationGuide/html/installguide.html) (optional, but required for typical purposes)
* [Geant4](http://geant4-userdoc.web.cern.ch/geant4-userdoc/UsersGuides/InstallationGuide/html/installguide.html) (optional, but required for typical purposes, multithreading capabilities should be enabled)
* [Eigen3](http://eigen.tuxfamily.org/index.php?title=Main_Page) (optional, but required for typical purposes)

## Usage on CERN machines

## Usage on machines with CVMFS

Ready-to-run installations of Allpix<sup>2</sup> are provided via the CERN Virtual Machine File System (CVMFS) for use on SLC6 and CentOS7 machines supported by CERN. In order to use Allpix<sup>2</sup>, the respective environment has to be set up by sourcing the appropriate file:

Expand All @@ -27,6 +32,7 @@ It should be noted that the CVMFS cache of the executing machine has to be popul
This can lead to a significant start-up time for the first execution, but should not affect further executions with the cache already present.
More information can be found in the [CVMFS documentation](https://cernvm.cern.ch/portal/filesystem).


## Installation
The CMake build system is used for compilation and installation. The install directory can be specified by adding `-DCMAKE_INSTALL_PREFIX=<prefix>` as argument to the CMake command below. Other configuration options are explained in the manual (see the documentation section below).

Expand All @@ -53,8 +59,9 @@ $ make install

For more detailed installation instructions, please refer to the documentation below.


## Documentation
The most recently published version of the User Manual is available online [here](https://cern.ch/allpix-squared/usermanual/allpix-manual.pdf).
The most recently published version of the User Manual is available [from the website](https://cern.ch/allpix-squared/usermanual/allpix-manual.pdf).
The Doxygen reference is published [online](https://cern.ch/allpix-squared/reference/) too.

The latest PDF version of the User Manual can also be created from source by executing
Expand All @@ -69,14 +76,14 @@ $ make reference
```
The main page of the reference can then be found at `reference/html/index.html` in the build folder.


## Development of Allpix<sup>2</sup>

Allpix<sup>2</sup> has been developed and is maintained by

* Koen Wolters, CERN, @kwolters
* Daniel Hynds, Nikhef, @dhynds
* Paul Schütze, DESY, @pschutze
* Simon Spannagel, DESY, @simonspa
* Koen Wolters, CERN, @kwolters

The following authors, in alphabetical order, have contributed to Allpix<sup>2</sup>:
* Mohamed Moanis Ali, Free University of Bozen-Bolzano, @mmoanis
Expand All @@ -90,6 +97,7 @@ The following authors, in alphabetical order, have contributed to Allpix<sup>2</
* Katharina Dort, University of Gie\ss en, @kdort
* Neal Gauvin, Université de Genève, @ngauvin
* Lennart Huth, DESY, @lhuth
* Daniel Hynds, University of Oxford, @dhynds
* Maoqiang Jing, University of South China, Institute of High Energy Physics Beijing, @mjing
* Moritz Kiehn, Université de Genève, @msmk
* Stephan Lachnit, Heidelberg University, @stephanlachnit
Expand All @@ -104,13 +112,13 @@ The following authors, in alphabetical order, have contributed to Allpix<sup>2</
* Enrico Jr. Schioppa, Unisalento and INFN Lecce, @schioppa
* Sanchit Sharma, Kansas State University, @SanchitKratos
* Xin Shi, Institute of High Energy Physics Beijing, @xshi
* Viktor Sonesten, GSOC18 Student, @tmplt
* Ondrej Theiner, Charles University, @otheiner
* Annika Vauth, University of Hamburg, @avauth
* Mateus Vicente Barreto Pinto, CERN, @mvicente
* Andy Wharton, Lancaster University, @awharton
* Morag Williams, University of Glasgow, @williamm

The authors would also like to express their thanks to the developers of [AllPix](https://twiki.cern.ch/twiki/bin/view/Main/AllPix).

## Citations
The reference paper of Allpix Squared describing the framework and providing first validation results using test beam data has been published in *Nuclear Instrumentations and Methods in Physics Research A*.
Expand All @@ -130,21 +138,23 @@ In addition, the software can be cited using the [versioned Zenodo record](https
> S. Spannagel, K. Wolters, D. Hynds. (2019, July 9). Allpix Squared - Generic Pixel Detector Simulation Framework
> (Version 1.4.0). Zenodo. http://doi.org/10.5281/zenodo.3550971

## Contributing
All types of contributions, being it minor and major, are very welcome. Please refer to our [contribution guidelines](CONTRIBUTING.md) for a description on how to get started.

Before adding changes it is very much recommended to carefully read through the documentation in the User Manual first.


## Licenses
This software is distributed under the terms of the MIT license. A copy of this license can be found in [LICENSE.md](LICENSE.md).

The documentation is distributed under the terms of the CC-BY-4.0 license. This license can be found in [doc/COPYING.md](doc/COPYING.md).

The LaTeX, Pandoc and PostgreSQL CMake modules used by Allpix<sup>2</sup> are licensed under the BSD 3-Clause License.
The CodeCoverage CMake module is copyright 2012 - 2017 by Lars Bilke, the full license can be found in the [module file](cmake/CodeCoverage.cmake).

The octree library of the TCAD DF-ISE converter is made available under the MIT license, more information [here](tools/tcad_dfise_converter/README.md).

The cereal C++11 serialization library used by Allpix<sup>2</sup> is published under the BSD 3-Clause License, the original source code can be found [here](https://github.com/USCiLab/cereal).
The following third-party codes are included in the repository:

The combination algorithms by Howard Hinnant are published under the Boost Software License Version 1.0, the code can be found [here](https://github.com/HowardHinnant/combinations) and the documentation of the class is published [here](https://howardhinnant.github.io/combinations/combinations.html).
* The LaTeX, Pandoc and PostgreSQL CMake modules used by Allpix<sup>2</sup> are licensed under the BSD 3-Clause License.
* The CodeCoverage CMake module is copyright 2012 - 2017 by Lars Bilke, the full license can be found in the [module file](cmake/CodeCoverage.cmake).
* The octree library of the TCAD DF-ISE converter is made available under the MIT license, more information [here](tools/tcad_dfise_converter/README.md).
* The cereal C++11 serialization library used by Allpix<sup>2</sup> is published under the BSD 3-Clause License, the original source code can be found [here](https://github.com/USCiLab/cereal).
* The combination algorithms by Howard Hinnant are published under the Boost Software License Version 1.0, the code can be found [here](https://github.com/HowardHinnant/combinations) and the documentation of the class is published [here](https://howardhinnant.github.io/combinations/combinations.html).
* The CTest to JUnit XSL template is published under the MIT License, the original source code can be found [here](https://github.com/rpavlik/jenkins-ctest-plugin).
5 changes: 5 additions & 0 deletions cmake/AllpixMacros.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ MACRO(allpix_module_install name)
ARCHIVE DESTINATION lib)
ENDMACRO()

MACRO(allpix_module_require_geant4_interface name)
SET(ALLPIX_BUILD_GEANT4_INTERFACE "ON" CACHE BOOL "Build Geant4 interface library" FORCE)
TARGET_LINK_LIBRARIES(${name} ${ALLPIX_GEANT4_INTERFACE})
ENDMACRO()

# Macro to set up Eigen3:: targets
MACRO(ALLPIX_SETUP_EIGEN_TARGETS)

Expand Down
Loading

0 comments on commit acb8629

Please sign in to comment.