Merge dev into main for v1.2 release

.clang-format

@@ -41,3 +41,6 @@ SpacesInContainerLiterals: 'false'
 SpacesInParentheses: 'false'
 SpacesInSquareBrackets: 'false'
 TabWidth: '4'
+---
+Language: Json
+BasedOnStyle: llvm

.gitlab-ci.yml

@@ -1,105 +1,134 @@
 stages:
+  - prepare
   - build
   - test
 
+workflow:
+  rules:
+    # Run pipeline for GitHub PRs to dev (does not support PRs from forks)
+    - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
+    - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "main"
+    - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_REF_PROTECTED == "true"
+    # Run pipeline if manually triggered via API or web GUI
+    - if: $CI_PIPELINE_SOURCE == "api"
+    - if: $CI_PIPELINE_SOURCE == "web"
+
+.load-modules:
+  before_script:
+    - ml vis/Graphviz/8.1.0-GCCcore-12.3.0 fpga xilinx/xrt/2.16 devel/Doxygen/1.9.7-GCCcore-12.3.0 compiler/GCC/12.3.0 devel/CMake/3.26.3-GCCcore-12.3.0
+
 build-dependencies:
-  stage: build
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: prepare
   variables:
-    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 1:00:00 -N 1 -n 1 --cpus-per-task=128"
+    #SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 2:00:00 -N 1 -n 1 --cpus-per-task=128"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc -t 2:00:00"
+  extends: .load-modules
   script:
-    - ml fpga
-    - ml xilinx/xrt/2.14
-    - ml devel/Doxygen/1.9.5-GCCcore-12.2.0
-    - ml compiler/GCC/12.2.0
-    - ml devel/CMake/3.24.3-GCCcore-12.2.0
     - ./buildDependencies.sh
   artifacts:
     paths:
       - "deps/"
     expire_in: 1 week
-  only:
-    refs:
-      - main
-      - dev
 
-build-job:
+build-unittests:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
   stage: build
   needs:
     - build-dependencies
   variables:
-    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 1:00:00 -N 1 -n 1 --cpus-per-task=128"
+    #SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 2:00:00 -N 1 -n 1 --cpus-per-task=128"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc -t 2:00:00"
+  extends: .load-modules
   script:
-    - ml vis/Graphviz/2.50.0-GCCcore-11.2.0
-    - ml fpga
-    - ml xilinx/xrt/2.14
-    - ml devel/Doxygen/1.9.5-GCCcore-12.2.0
-    - ml compiler/GCC/12.2.0
-    - ml devel/CMake/3.24.3-GCCcore-12.2.0
     - mkdir build && cd build
-    - cmake -DCMAKE_BUILD_TYPE=Release ..
-    - make -j $procs
+    - cmake -DCMAKE_BUILD_TYPE=Release -DFINN_BUILD_DOC=OFF -DFINN_ENABLE_SANITIZERS=ON ..
+    - make -j $(nproc)
   artifacts:
     paths:
       - "build/"
     expire_in: 1 week
-  only:
-    refs:
-      - main
-      - dev
 
 build-integrationtests:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: build
+  needs:
+    - build-dependencies
+  variables:
+    #SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 2:00:00 -N 1 -n 1 --cpus-per-task=128"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc -t 2:00:00"
+  extends: .load-modules
+  script:
+    - mkdir build-host-mem && cd build-host-mem
+    - cmake -DCMAKE_BUILD_TYPE=Release -DFINN_BUILD_DOC=OFF -DFINN_ENABLE_SANITIZERS=ON ..
+    - make Integrationtests -j $(nproc)
+    - cd ..
+    - mkdir build-no-host-mem && cd build-no-host-mem
+    - cmake -DCMAKE_BUILD_TYPE=Release -DFINN_USE_HOST_MEM=OFF -DFINN_BUILD_DOC=OFF -DFINN_ENABLE_SANITIZERS=ON ..
+    - make Integrationtests -j $(nproc)
+  artifacts:
+    paths:
+      - "build-host-mem/"
+      - "build-no-host-mem/"
+    expire_in: 1 week
+
+build-regressiontests:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
   stage: build
   needs:
     - build-dependencies
   variables:
-    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 1:00:00 -N 1 -n 1 --cpus-per-task=128"
+    #SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 2:00:00 -N 1 -n 1 --cpus-per-task=128"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc -t 2:00:00"
+  extends: .load-modules
   script:
-    - ml vis/Graphviz/2.50.0-GCCcore-11.2.0
-    - ml fpga
-    - ml xilinx/xrt/2.14
-    - ml devel/Doxygen/1.9.5-GCCcore-12.2.0
-    - ml compiler/GCC/12.2.0
-    - ml devel/CMake/3.24.3-GCCcore-12.2.0
     - mkdir build && cd build
-    - cmake -DCMAKE_BUILD_TYPE=Release ..
-    - make Integrationtests -j $procs
+    - cmake -DCMAKE_BUILD_TYPE=Release -DFINN_BUILD_DOC=OFF -DFINN_ENABLE_SANITIZERS=OFF ..
+    - make -j $(nproc)
   artifacts:
     paths:
       - "build/"
     expire_in: 1 week
-  only:
-    refs:
-      - main
-      - dev
 
 lint:
-  stage: test
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: prepare
   variables:
-    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 0:30:00 -N 1 -n 1 --cpus-per-task=2 --mem-per-cpu=2G"
+    #SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 0:30:00 -N 1 -n 1 --cpus-per-task=2 --mem-per-cpu=2G"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc -t 2:00:00"
   script:
     - ml lang
     - ml Python/3.10.4-GCCcore-11.3.0
     - command -v clang-format >/dev/null 2>&1 || pip3 install --user clang-format
     - find ./src \( -name \*.cpp -o -name \*.h -o -name \*.hpp \) -print0 | xargs -0 -n 1 /pc2/users/e/ekiappci/.local/bin/clang-format -style=file --Werror -n --verbose
     - find ./unittests \( -name \*.cpp -o -name \*.h -o -name \*.hpp \) -print0 | xargs -0 -n 1 /pc2/users/e/ekiappci/.local/bin/clang-format -style=file --Werror -n --verbose
     - find ./benchmarks \( -name \*.cpp -o -name \*.h -o -name \*.hpp \) -print0 | xargs -0 -n 1 /pc2/users/e/ekiappci/.local/bin/clang-format -style=file --Werror -n --verbose
-  only:
-    refs:
-      - main
-      - dev
 
 run-unittests:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
   stage: test
-  dependencies:
-    - build-job
+  needs:
+    - build-unittests
     - build-dependencies
   variables:
-    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 0:30:00 -N 1 -n 1 --cpus-per-task=2 --mem-per-cpu=2G"
+    #SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 0:30:00 -N 1 -n 1 --cpus-per-task=2 --mem-per-cpu=2G"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc -t 2:00:00"
+  extends: .load-modules
   script:
-    - ml fpga
-    - ml xilinx/xrt/2.14
-    - ml compiler/GCC/12.2.0
-    - ml devel/CMake/3.24.3-GCCcore-12.2.0
+    - export LD_LIBRARY_PATH="$(pwd)/deps/finn_boost/stage/lib/boost/:$LD_LIBRARY_PATH"
     - cd build
     - ctest --output-on-failure --output-junit ctest-results.xml
   artifacts:
@@ -108,36 +137,76 @@ run-unittests:
     expire_in: 1 week
     reports:
       junit: build/ctest-results.xml
-  only:
-    refs:
-      - main
-      - dev
 
 
 run-integrationtests:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
   stage: test
-  dependencies:
+  needs:
     - build-integrationtests
     - build-dependencies
   variables:
-    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p fpga  -t 0:30:00 --constraint=xilinx_u280_xrt2.14"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc  -t 0:30:00"
+  extends: .load-modules
   script:
-    - ml fpga
-    - ml xilinx/xrt/2.14
-    - ml compiler/GCC/12.2.0
-    - ml devel/CMake/3.24.3-GCCcore-12.2.0
-    - cp example_networks/identity_net/* build/integrationtest
-    - cd build/integrationtest
-    - xbutil reset -d 0000:a1:00.1
+    - export LD_LIBRARY_PATH="$(pwd)/deps/finn_boost/stage/lib/boost/:$LD_LIBRARY_PATH"
+    # Reset FPGAs and setup Host Memory Access
+    - ml xilinx/u55c/u55c_gen3x16_xdma_3_202210_1
+    - xbutil reset -d 0000:c1:00.1
     - xbutil reset -d 0000:81:00.1
-    - xbutil reset -d 0000:01:00.1
+    - sudo /opt/software/FPGA/scripts/u280/host_memory/enable_hostmem_single_fpga.sh 0000:c1:00.1
+    - /opt/software/FPGA/scripts/u280/host_memory/verify_single_fpga.sh 0000:c1:00.1
+    - sudo /opt/software/FPGA/scripts/u280/host_memory/enable_hostmem_single_fpga.sh 0000:81:00.1
+    - /opt/software/FPGA/scripts/u280/host_memory/verify_single_fpga.sh 0000:81:00.1
+
+    - cd build-host-mem/integrationtest
+    # Test Synchronous Inference with Host memory
+    - cp ../../example_networks/jet-structure-classification-with-host-mem/* .
     - ./SyncInference
-    # - xbutil reset -d 0000:a1:00.1
-    # - xbutil reset -d 0000:81:00.1
-    # - xbutil reset -d 0000:01:00.1
-    # - ./AsyncInference
-  only:
-    refs:
-      - main
-      - dev
 
+    # Reset FPGAs again
+    - xbutil reset -d 0000:c1:00.1
+    - xbutil reset -d 0000:81:00.1
+
+    # Test Synchronous Inference without Host memory
+    - cd ../../build-no-host-mem/integrationtest
+    - cp ../../example_networks/jet-structure-classification-without-host-mem/* .
+    - ./SyncInference
+
+    # TODO: Run regression tests + asynchronous inference
+
+run-regressiontests:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: test
+  needs:
+    - build-regressiontests
+    - build-dependencies
+  variables:
+    #SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p normal -t 0:30:00 -N 1 -n 1 --cpus-per-task=2 --mem-per-cpu=2G"
+    SCHEDULER_PARAMETERS: "-A hpc-prf-ekiapp -p hacc -t 0:30:00"
+  extends: .load-modules
+  script:
+    - export LD_LIBRARY_PATH="$(pwd)/deps/finn_boost/stage/lib/boost/:$LD_LIBRARY_PATH"
+    # Reset FPGAs and setup Host Memory Access
+    - ml xilinx/u55c/u55c_gen3x16_xdma_3_202210_1
+    - xbutil reset -d 0000:c1:00.1
+    - xbutil reset -d 0000:81:00.1
+    - sudo /opt/software/FPGA/scripts/u280/host_memory/enable_hostmem_single_fpga.sh 0000:c1:00.1
+    - /opt/software/FPGA/scripts/u280/host_memory/verify_single_fpga.sh 0000:c1:00.1
+    - sudo /opt/software/FPGA/scripts/u280/host_memory/enable_hostmem_single_fpga.sh 0000:81:00.1
+    - /opt/software/FPGA/scripts/u280/host_memory/verify_single_fpga.sh 0000:81:00.1
+
+    - cd build/benchmarks
+    - cp ../../example_networks/jet-structure-classification-with-host-mem/* .
+    - ../_deps/googlebenchmark-src/tools/compare.py -a -d results.json benchmarks ../../benchmarks/expectedPerformance.json ./SynchronousInferenceBenchmark | tee benchmark_output.txt
+    # Extract the OVERALL_GEOMEAN p-value and check if it exceeds the threshold
+    - THRESHOLD=0.05
+    - GEOMEAN=$(grep "OVERALL_GEOMEAN" benchmark_output.txt)
+    - if [[ -z "$GEOMEAN" ]]; then exit 1;fi
+    - VALUE=$(echo "$GEOMEAN" | awk '{print $3}' | sed 's/\x1B\[[0-9;]\{1,\}[A-Za-z]//g')
+    - if (( $(echo "$VALUE > $THRESHOLD" | bc -l) )); then exit 1;fi
+