From a1c8af320bab598699a9ab5c9c343df5af17f0a2 Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Wed, 20 Sep 2023 16:18:07 +0100 Subject: [PATCH 01/14] New settings file format --- doc/{exclude-patterns.inc => exclude_patterns.inc} | 2 -- settings.json | 5 ----- settings.yml | 14 ++++++++++++++ 3 files changed, 14 insertions(+), 7 deletions(-) rename doc/{exclude-patterns.inc => exclude_patterns.inc} (85%) delete mode 100644 settings.json create mode 100644 settings.yml diff --git a/doc/exclude-patterns.inc b/doc/exclude_patterns.inc similarity index 85% rename from doc/exclude-patterns.inc rename to doc/exclude_patterns.inc index e568e88a..b3fff549 100644 --- a/doc/exclude-patterns.inc +++ b/doc/exclude_patterns.inc @@ -8,7 +8,5 @@ build/* **/*.md # We need to ensure these are not processed twice which breaks figure numbering. They are included via index.rst. -**/multi_rate_hifi_src.rst -**/fixed_ratio_src.rst **/resource_usage_asrc.rst **/resource_usage_ssrc.rst \ No newline at end of file diff --git a/settings.json b/settings.json deleted file mode 100644 index 78a4bf84..00000000 --- a/settings.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "title": "SAMPLE RATE CONVERSION", - "project": "lib_src documentation", - "version": "2.3.0" -} diff --git a/settings.yml b/settings.yml new file mode 100644 index 00000000..439a04cf --- /dev/null +++ b/settings.yml @@ -0,0 +1,14 @@ +--- +project: lib_src +title: SAMPLE RATE CONVERSION +version: 2.3.0 + +documentation: + exclude_patterns_path: doc/exclude_patterns.inc + doxygen_projects: + lib_xcore_math: + doxyfile_path: doc/Doxyfile.inc + pdfs: + doc/programming_guide/index: + pdf_title: "{{title}} - Programming Guide" + pdf_filename: "{{project}}_programming_guide_v{{version}}" From e64b91ce5ad2705983e5cd1c320e2a46944af739 Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Wed, 20 Sep 2023 17:34:27 +0100 Subject: [PATCH 02/14] Move docs out of "rst" folder Also replace multiple includes with toc for programming guide --- doc/exclude_patterns.inc | 1 + .../programming_guide/fixed_ratio_src.rst | 2 ++ .../images/asrc_fft_176_48.pdf | Bin .../images/asrc_fft_44_192.pdf | Bin .../images/asrc_fft_96_88.pdf | Bin .../images/asrc_structure.pdf | Bin .../images/quad_dual_instance.pdf | Bin .../programming_guide/images/src_ff3_vpu.png | Bin .../images/src_ff3_vpu_pb.png | Bin .../programming_guide/images/src_proc.pdf | Bin .../programming_guide/images/src_rat_vpu.png | Bin .../images/src_rat_vpu_pb.png | Bin .../images/ssrc_fft_176_48.pdf | Bin .../images/ssrc_fft_44_192.pdf | Bin .../images/ssrc_fft_96_88.pdf | Bin .../images/ssrc_structure.pdf | Bin .../images/stereo_dual_instance.pdf | Bin .../images/stereo_single_instance.pdf | Bin doc/programming_guide/index.rst | 9 ++++++ .../programming_guide/multi_rate_hifi_src.rst | 10 +++---- .../programming_guide/resource_usage_asrc.rst | 0 .../programming_guide/resource_usage_ssrc.rst | 0 doc/rst/programming_guide/index.rst | 10 ------- doc/rst/version.rst | 1 - doc/src_html/custom.css | 27 ------------------ index.rst | 2 +- 26 files changed, 18 insertions(+), 44 deletions(-) rename doc/{rst => }/programming_guide/fixed_ratio_src.rst (99%) rename doc/{rst => }/programming_guide/images/asrc_fft_176_48.pdf (100%) rename doc/{rst => }/programming_guide/images/asrc_fft_44_192.pdf (100%) rename doc/{rst => }/programming_guide/images/asrc_fft_96_88.pdf (100%) rename doc/{rst => }/programming_guide/images/asrc_structure.pdf (100%) rename doc/{rst => }/programming_guide/images/quad_dual_instance.pdf (100%) rename doc/{rst => }/programming_guide/images/src_ff3_vpu.png (100%) rename doc/{rst => }/programming_guide/images/src_ff3_vpu_pb.png (100%) rename doc/{rst => }/programming_guide/images/src_proc.pdf (100%) rename doc/{rst => }/programming_guide/images/src_rat_vpu.png (100%) rename doc/{rst => }/programming_guide/images/src_rat_vpu_pb.png (100%) rename doc/{rst => }/programming_guide/images/ssrc_fft_176_48.pdf (100%) rename doc/{rst => }/programming_guide/images/ssrc_fft_44_192.pdf (100%) rename doc/{rst => }/programming_guide/images/ssrc_fft_96_88.pdf (100%) rename doc/{rst => }/programming_guide/images/ssrc_structure.pdf (100%) rename doc/{rst => }/programming_guide/images/stereo_dual_instance.pdf (100%) rename doc/{rst => }/programming_guide/images/stereo_single_instance.pdf (100%) create mode 100644 doc/programming_guide/index.rst rename doc/{rst => }/programming_guide/multi_rate_hifi_src.rst (99%) rename doc/{rst => }/programming_guide/resource_usage_asrc.rst (100%) rename doc/{rst => }/programming_guide/resource_usage_ssrc.rst (100%) delete mode 100644 doc/rst/programming_guide/index.rst delete mode 100644 doc/rst/version.rst delete mode 100644 doc/src_html/custom.css diff --git a/doc/exclude_patterns.inc b/doc/exclude_patterns.inc index b3fff549..599c2e6c 100644 --- a/doc/exclude_patterns.inc +++ b/doc/exclude_patterns.inc @@ -1,6 +1,7 @@ # The following patterns are to be excluded from the documentation build tests/* build/* +**LICENSE* # Do not build .md files diff --git a/doc/rst/programming_guide/fixed_ratio_src.rst b/doc/programming_guide/fixed_ratio_src.rst similarity index 99% rename from doc/rst/programming_guide/fixed_ratio_src.rst rename to doc/programming_guide/fixed_ratio_src.rst index 26e9afea..9e8d62f0 100644 --- a/doc/rst/programming_guide/fixed_ratio_src.rst +++ b/doc/programming_guide/fixed_ratio_src.rst @@ -121,7 +121,9 @@ They have been designed for voice applications and, in particular, conformance t .. note:: These filters will only run on XCORE-AI due to the inner dot product calculation employing the XS3 VPU. + .. _ff3_voice_vpu_hdr: + Fixed factor of 3 VPU --------------------- diff --git a/doc/rst/programming_guide/images/asrc_fft_176_48.pdf b/doc/programming_guide/images/asrc_fft_176_48.pdf similarity index 100% rename from doc/rst/programming_guide/images/asrc_fft_176_48.pdf rename to doc/programming_guide/images/asrc_fft_176_48.pdf diff --git a/doc/rst/programming_guide/images/asrc_fft_44_192.pdf b/doc/programming_guide/images/asrc_fft_44_192.pdf similarity index 100% rename from doc/rst/programming_guide/images/asrc_fft_44_192.pdf rename to doc/programming_guide/images/asrc_fft_44_192.pdf diff --git a/doc/rst/programming_guide/images/asrc_fft_96_88.pdf b/doc/programming_guide/images/asrc_fft_96_88.pdf similarity index 100% rename from doc/rst/programming_guide/images/asrc_fft_96_88.pdf rename to doc/programming_guide/images/asrc_fft_96_88.pdf diff --git a/doc/rst/programming_guide/images/asrc_structure.pdf b/doc/programming_guide/images/asrc_structure.pdf similarity index 100% rename from doc/rst/programming_guide/images/asrc_structure.pdf rename to doc/programming_guide/images/asrc_structure.pdf diff --git a/doc/rst/programming_guide/images/quad_dual_instance.pdf b/doc/programming_guide/images/quad_dual_instance.pdf similarity index 100% rename from doc/rst/programming_guide/images/quad_dual_instance.pdf rename to doc/programming_guide/images/quad_dual_instance.pdf diff --git a/doc/rst/programming_guide/images/src_ff3_vpu.png b/doc/programming_guide/images/src_ff3_vpu.png similarity index 100% rename from doc/rst/programming_guide/images/src_ff3_vpu.png rename to doc/programming_guide/images/src_ff3_vpu.png diff --git a/doc/rst/programming_guide/images/src_ff3_vpu_pb.png b/doc/programming_guide/images/src_ff3_vpu_pb.png similarity index 100% rename from doc/rst/programming_guide/images/src_ff3_vpu_pb.png rename to doc/programming_guide/images/src_ff3_vpu_pb.png diff --git a/doc/rst/programming_guide/images/src_proc.pdf b/doc/programming_guide/images/src_proc.pdf similarity index 100% rename from doc/rst/programming_guide/images/src_proc.pdf rename to doc/programming_guide/images/src_proc.pdf diff --git a/doc/rst/programming_guide/images/src_rat_vpu.png b/doc/programming_guide/images/src_rat_vpu.png similarity index 100% rename from doc/rst/programming_guide/images/src_rat_vpu.png rename to doc/programming_guide/images/src_rat_vpu.png diff --git a/doc/rst/programming_guide/images/src_rat_vpu_pb.png b/doc/programming_guide/images/src_rat_vpu_pb.png similarity index 100% rename from doc/rst/programming_guide/images/src_rat_vpu_pb.png rename to doc/programming_guide/images/src_rat_vpu_pb.png diff --git a/doc/rst/programming_guide/images/ssrc_fft_176_48.pdf b/doc/programming_guide/images/ssrc_fft_176_48.pdf similarity index 100% rename from doc/rst/programming_guide/images/ssrc_fft_176_48.pdf rename to doc/programming_guide/images/ssrc_fft_176_48.pdf diff --git a/doc/rst/programming_guide/images/ssrc_fft_44_192.pdf b/doc/programming_guide/images/ssrc_fft_44_192.pdf similarity index 100% rename from doc/rst/programming_guide/images/ssrc_fft_44_192.pdf rename to doc/programming_guide/images/ssrc_fft_44_192.pdf diff --git a/doc/rst/programming_guide/images/ssrc_fft_96_88.pdf b/doc/programming_guide/images/ssrc_fft_96_88.pdf similarity index 100% rename from doc/rst/programming_guide/images/ssrc_fft_96_88.pdf rename to doc/programming_guide/images/ssrc_fft_96_88.pdf diff --git a/doc/rst/programming_guide/images/ssrc_structure.pdf b/doc/programming_guide/images/ssrc_structure.pdf similarity index 100% rename from doc/rst/programming_guide/images/ssrc_structure.pdf rename to doc/programming_guide/images/ssrc_structure.pdf diff --git a/doc/rst/programming_guide/images/stereo_dual_instance.pdf b/doc/programming_guide/images/stereo_dual_instance.pdf similarity index 100% rename from doc/rst/programming_guide/images/stereo_dual_instance.pdf rename to doc/programming_guide/images/stereo_dual_instance.pdf diff --git a/doc/rst/programming_guide/images/stereo_single_instance.pdf b/doc/programming_guide/images/stereo_single_instance.pdf similarity index 100% rename from doc/rst/programming_guide/images/stereo_single_instance.pdf rename to doc/programming_guide/images/stereo_single_instance.pdf diff --git a/doc/programming_guide/index.rst b/doc/programming_guide/index.rst new file mode 100644 index 00000000..0e8c0b41 --- /dev/null +++ b/doc/programming_guide/index.rst @@ -0,0 +1,9 @@ +Programming Guide +################# + +.. toctree:: + + ../../README + multi_rate_hifi_src + fixed_ratio_src + ../../CHANGELOG diff --git a/doc/rst/programming_guide/multi_rate_hifi_src.rst b/doc/programming_guide/multi_rate_hifi_src.rst similarity index 99% rename from doc/rst/programming_guide/multi_rate_hifi_src.rst rename to doc/programming_guide/multi_rate_hifi_src.rst index 0a7cc6da..17d12ce0 100644 --- a/doc/rst/programming_guide/multi_rate_hifi_src.rst +++ b/doc/programming_guide/multi_rate_hifi_src.rst @@ -46,11 +46,11 @@ There is an initialization call which sets up the variables within the structure Initialization ensures the correct selection, ordering and configuration of the filtering stages, be they decimators, interpolators or pass-through blocks. This initialization call contains arguments defining selected input and output nominal sample rates as well as settings for the sample rate converter: -.. doxygenfunction:: ssrc_init +:c:func:`ssrc_init` The initialization call is the same for ASRC: -.. doxygenfunction:: asrc_init +:c:func:`asrc_init` The input block size must be a power of 2 and is set by the ``n_in_samples`` argument. In the case where more than one channel is to be processed per SRC instance, the total number of input samples expected for each processing call is ``n_in_samples * n_channels_per_instance``. @@ -75,11 +75,11 @@ The logic is designed so that the final filtering stage always receives a sample The processing function call is passed the input and output buffers and a reference to the control structure: -.. doxygenfunction:: ssrc_process +:c:func:`ssrc_process` In the case of ASRC a fractional frequency ratio argument is also supplied: -.. doxygenfunction:: asrc_process +:c:func:`asrc_process` The SRC processing call always returns a whole number of output samples produced by the sample rate conversion. Depending on the sample ratios selected, this number may be between zero and ``(n_in_samples * n_channels_per_instance * SRC_N_OUT_IN_RATIO_MAX)``. ``SRC_N_OUT_IN_RATIO_MAX`` is the maximum number of output samples for a single input sample. For example, if the input frequency is 44.1 kHz and the output rate is 192 kHz then a sample rate conversion of one sample input may produce up to 5 output samples. @@ -477,4 +477,4 @@ ASRC API -------- .. doxygengroup:: src_asrc - :content-only: \ No newline at end of file + :content-only: diff --git a/doc/rst/programming_guide/resource_usage_asrc.rst b/doc/programming_guide/resource_usage_asrc.rst similarity index 100% rename from doc/rst/programming_guide/resource_usage_asrc.rst rename to doc/programming_guide/resource_usage_asrc.rst diff --git a/doc/rst/programming_guide/resource_usage_ssrc.rst b/doc/programming_guide/resource_usage_ssrc.rst similarity index 100% rename from doc/rst/programming_guide/resource_usage_ssrc.rst rename to doc/programming_guide/resource_usage_ssrc.rst diff --git a/doc/rst/programming_guide/index.rst b/doc/rst/programming_guide/index.rst deleted file mode 100644 index 9453a906..00000000 --- a/doc/rst/programming_guide/index.rst +++ /dev/null @@ -1,10 +0,0 @@ -Programming Guide -################# - -.. include:: ../../../README.rst - -.. include:: ./multi_rate_hifi_src.rst - -.. include:: ./fixed_ratio_src.rst - -.. include:: ../../../CHANGELOG.rst diff --git a/doc/rst/version.rst b/doc/rst/version.rst deleted file mode 100644 index 06defed3..00000000 --- a/doc/rst/version.rst +++ /dev/null @@ -1 +0,0 @@ -.. version:: 2.3.0 diff --git a/doc/src_html/custom.css b/doc/src_html/custom.css deleted file mode 100644 index 8bca20fb..00000000 --- a/doc/src_html/custom.css +++ /dev/null @@ -1,27 +0,0 @@ -/* no longer needed */ -/* This rule allows scrolling the navigation menu on mobile -@media (max-width: 67em) { - .sidebar-drawer { - overflow-y: scroll; - } -} */ - -/* no longer needed */ -/* this helps with the sticky sidebar scrolling focus issues -.sidebar-sticky, .toc-sticky { - height: auto; -} */ - - -h1, h2, h3, h4, h5, h6{ - font-weight: 300; -} - -table.docutils td p{ - font-size: x-small -} - -table.docutils th p{ - font-size: small; - font-weight: 300; -} \ No newline at end of file diff --git a/index.rst b/index.rst index 14e6c13e..f51d77dd 100644 --- a/index.rst +++ b/index.rst @@ -6,4 +6,4 @@ SAMPLE RATE CONVERSION .. toctree:: :maxdepth: 5 - ./doc/rst/programming_guide/index.rst \ No newline at end of file + ./doc/programming_guide/index.rst From 893fe65f6fcfd7ce4aa222ad5094afbe2550bc4f Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Wed, 20 Sep 2023 17:36:12 +0100 Subject: [PATCH 03/14] Switch from github actions to jenkins for doc build --- .github/workflows/docs.yml | 57 ---------- Jenkinsfile | 209 +++++++++++++++++++++---------------- 2 files changed, 118 insertions(+), 148 deletions(-) delete mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml deleted file mode 100644 index 25be91a1..00000000 --- a/.github/workflows/docs.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Documentation - -on: - schedule: - # run at 5am every day - - cron: '0 5 * * *' - push: - paths: - - '.github/workflows/docs.yml' - - 'settings.json' - - 'doc/exclude_patterns.inc' - - '**/doc/**' - pull_request: - paths: - - '.github/workflows/docs.yml' - - 'settings.json' - - 'doc/exclude-patterns.inc' - - '**/doc/**' - - # Allow manually triggering the workflow. - workflow_dispatch: {} - -env: - XCORE_DOC_BUILDER: 'ghcr.io/xmos/doc_builder:v3.0.0' - -jobs: - build_documentation: - name: Build and package documentation - if: github.repository_owner == 'xmos' - runs-on: ubuntu-latest - steps: - - name: Checkout this repo - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: '3.10.x' - - - name: Pull doc_builder container - run: | - docker pull ${XCORE_DOC_BUILDER} - - - name: Build documentation - run: | - pwd - docker run --user "$(id -u):$(id -g)" --rm -v ${{ github.workspace }}:/build -e EXCLUDE_PATTERNS="/build/doc/exclude-patterns.inc" -e OUTPUT_DIR="/build/doc/_build" -e PDF=1 -e SKIP_LINK=1 -e DOXYGEN_INCLUDE=/build/doc/Doxyfile.inc -e DOXYGEN_INPUT=ignore ${XCORE_DOC_BUILDER} - tree - DOC_VERSION=$(grep version settings.json | grep -o "[0-9]*\.[0-9]*\.[0-9]") - mv doc/_build/pdf/programming_guide.pdf doc/_build/pdf/programming_guide_v${DOC_VERSION}.pdf - - - name: Save documentation artifacts - uses: actions/upload-artifact@v3 - with: - name: docs lib_src - path: doc/_build - if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` - retention-days: 30 diff --git a/Jenkinsfile b/Jenkinsfile index 4b9e3175..ec827138 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -28,6 +28,7 @@ pipeline { } options { skipDefaultCheckout() + timestamps() } parameters { string( @@ -37,113 +38,139 @@ pipeline { ) } stages { - stage('Get repo') { - steps { - sh "mkdir ${REPO}" - // source checks require the directory - // name to be the same as the repo name - dir("${REPO}") { - // checkout repo - checkout scm - sh 'git submodule update --init --recursive --depth 1' - } - } - } - stage ("Create Python environment") - { - steps { - dir("${REPO}") { - createVenv('requirements.txt') - withVenv { - sh 'pip install -r requirements.txt' - } - } - } - } - stage('Library checks') { - steps { - dir("${REPO}") { - sh 'git clone git@github.com:xmos/infr_apps.git' - sh 'git clone git@github.com:xmos/infr_scripts_py.git' - // These are needed for xmake legacy build and also changelog check - sh 'git clone git@github.com:xmos/lib_logging.git' - sh 'git clone git@github.com:xmos/lib_xassert.git' - withVenv { - sh 'pip install -e infr_scripts_py' - sh 'pip install -e infr_apps' - dir("tests") { - withEnv(["XMOS_ROOT=.."]) { - localRunPytest('-s test_lib_checks.py -vv') + stage('Build and Docs') { + parallel { + stage('Build and Test') { + stages { + stage('Get repo') { + steps { + sh "mkdir ${REPO}" + // source checks require the directory + // name to be the same as the repo name + dir("${REPO}") { + // checkout repo + checkout scm + sh 'git submodule update --init --recursive --depth 1' + } } } - } - } - } - } - stage('Test xmake build') { - steps { - runningOn(env.NODE_NAME) - dir("${REPO}") { - withTools(params.TOOLS_VERSION) { - withVenv { - dir("tests") { - localRunPytest('-k "legacy" -vv') + stage ("Create Python environment") { + steps { + dir("${REPO}") { + createVenv('requirements.txt') + withVenv { + sh 'pip install -r requirements.txt' + } + } } } - } - } - } - } - stage('Tests XS2') { - steps { - runningOn(env.NODE_NAME) - dir("${REPO}") { - withTools(params.TOOLS_VERSION) { - withVenv { - sh 'mkdir build' - dir("build") { - sh 'rm -rf' - sh 'cmake --toolchain ../xmos_cmake_toolchain/xs2a.cmake ..' - sh 'make test_ds3_voice test_us3_voice test_unity_gain_voice -j' + stage('Library checks') { + steps { + dir("${REPO}") { + sh 'git clone git@github.com:xmos/infr_apps.git' + sh 'git clone git@github.com:xmos/infr_scripts_py.git' + // These are needed for xmake legacy build and also changelog check + sh 'git clone git@github.com:xmos/lib_logging.git' + sh 'git clone git@github.com:xmos/lib_xassert.git' + withVenv { + sh 'pip install -e infr_scripts_py' + sh 'pip install -e infr_apps' + dir("tests") { + withEnv(["XMOS_ROOT=.."]) { + localRunPytest('-s test_lib_checks.py -vv') + } + } + } + } } - dir("tests") { - localRunPytest('-n auto -k "xs2" -vv') + } + stage('Test xmake build') { + steps { + runningOn(env.NODE_NAME) + dir("${REPO}") { + withTools(params.TOOLS_VERSION) { + withVenv { + dir("tests") { + localRunPytest('-k "legacy" -vv') + } + } + } + } } - dir("build") { - sh 'rm -rf' // Cleanup XS2 cmake cache for next stage + } + stage('Tests XS2') { + steps { + runningOn(env.NODE_NAME) + dir("${REPO}") { + withTools(params.TOOLS_VERSION) { + withVenv { + sh 'mkdir build' + dir("build") { + sh 'rm -rf' + sh 'cmake --toolchain ../xmos_cmake_toolchain/xs2a.cmake ..' + sh 'make test_ds3_voice test_us3_voice test_unity_gain_voice -j' + } + dir("tests") { + localRunPytest('-n auto -k "xs2" -vv') + } + dir("build") { + sh 'rm -rf' // Cleanup XS2 cmake cache for next stage + } + } + } + } } } - } - } - } - } - stage('Tests XS3') { - steps { - runningOn(env.NODE_NAME) - dir("${REPO}") { - withTools(params.TOOLS_VERSION) { - withVenv { - dir("tests") { - localRunPytest('-m prepare') // Do all pre work like building and generating golden ref where needed + stage('Tests XS3') { + steps { + runningOn(env.NODE_NAME) + dir("${REPO}") { + withTools(params.TOOLS_VERSION) { + withVenv { + dir("tests") { + localRunPytest('-m prepare') // Do all pre work like building and generating golden ref where needed - // FF3 HiFi tests for OS3 and DS3 - localRunPytest('-m main -n auto -k "hifi_ff3" -vv') + // FF3 HiFi tests for OS3 and DS3 + localRunPytest('-m main -n auto -k "hifi_ff3" -vv') - // ASRC and SSRC tests across all in/out freqs and deviations (asrc only) - localRunPytest('-m main -n auto -k "mrhf" -vv') - archiveArtifacts artifacts: "mips_report*.csv", allowEmptyArchive: true + // ASRC and SSRC tests across all in/out freqs and deviations (asrc only) + localRunPytest('-m main -n auto -k "mrhf" -vv') + archiveArtifacts artifacts: "mips_report*.csv", allowEmptyArchive: true - // VPU enabled ff3 and rat tests - localRunPytest('-m main -k "vpu" -vv') // xdist not working yet so no -n auto + // VPU enabled ff3 and rat tests + localRunPytest('-m main -k "vpu" -vv') // xdist not working yet so no -n auto - // Profile the ASRC - localRunPytest('-m main -k "profile_asrc" -vv') - sh 'tree' - archiveArtifacts artifacts: "gprof_results/*.png", allowEmptyArchive: true + // Profile the ASRC + localRunPytest('-m main -k "profile_asrc" -vv') + sh 'tree' + archiveArtifacts artifacts: "gprof_results/*.png", allowEmptyArchive: true + } + } + } + } } } } } + stage('Build Docs') { + agent { label "docker" } + environment { XMOSDOC_VERSION = "pr-67" } + steps { + checkout scm + sh 'git submodule update --init --recursive --depth 1' + sh "docker pull ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION" + sh """docker run -u "\$(id -u):\$(id -g)" \ + --rm \ + -v ${WORKSPACE}:/build \ + ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION""" + archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true + } + post { + cleanup { + xcoreCleanSandbox() + } + } + } } } } From 5d4448fad964e2bfda0b582a56ce95b4282d4a2c Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Thu, 21 Sep 2023 10:19:26 +0100 Subject: [PATCH 04/14] Add PR label cheecking to enable doc-only builds --- Jenkinsfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index ec827138..bcf356db 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,6 +41,9 @@ pipeline { stage('Build and Docs') { parallel { stage('Build and Test') { + when { + expression { !env.GH_LABEL_DOC_ONLY.toBoolean() } + } stages { stage('Get repo') { steps { From 82443c9226a5c9d38410de91538efc38a8b81d42 Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Thu, 21 Sep 2023 10:35:10 +0100 Subject: [PATCH 05/14] Fix settings file name in lib test also make doc build verbose --- Jenkinsfile | 2 +- tests/test_lib_checks.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index bcf356db..c0e02100 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -165,7 +165,7 @@ pipeline { sh """docker run -u "\$(id -u):\$(id -g)" \ --rm \ -v ${WORKSPACE}:/build \ - ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION""" + ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true } post { diff --git a/tests/test_lib_checks.py b/tests/test_lib_checks.py index 02413ad1..07787342 100644 --- a/tests/test_lib_checks.py +++ b/tests/test_lib_checks.py @@ -36,9 +36,9 @@ def test_version_matches(): """ Check docs version vs changelog """ - with open(Path(__file__).resolve().parent / "../settings.json") as sj: + with open(Path(__file__).resolve().parent / "../settings.yml") as sy: with open(Path(__file__).resolve().parent / "../CHANGELOG.rst") as cl: re_string = r"([0-9]*)\.([0-9]*)\.([0-9]*)" - sj_ver = re.search(re_string, sj.readlines()[3]).groups() + sy_ver = re.search(re_string, sy.readlines()[3]).groups() cl_ver = re.search(re_string, cl.readlines()[3]).groups() - assert sj_ver == cl_ver, f"Version match issue between settings.json and CHANGELOG.rst: {sj_ver} {cl_ver}" + assert sy_ver == cl_ver, f"Version match issue between settings.yml and CHANGELOG.rst: {sy_ver} {cl_ver}" From 841faa75e69a04f1a6d5045a0ac554b2e8a7f28d Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Thu, 21 Sep 2023 10:47:23 +0100 Subject: [PATCH 06/14] use agent none at top level this means that we don't take up 2 agents --- Jenkinsfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c0e02100..adf746ac 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,9 +17,7 @@ def localRunPytest(String extra_args="") { getApproval() pipeline { - agent { - label 'x86_64&&macOS' // These agents have 24 cores so good for parallel xsim runs - } + agent none environment { REPO = 'lib_src' VIEW = getViewName(REPO) @@ -44,6 +42,9 @@ pipeline { when { expression { !env.GH_LABEL_DOC_ONLY.toBoolean() } } + agent { + label 'x86_64&&macOS' // These agents have 24 cores so good for parallel xsim runs + } stages { stage('Get repo') { steps { From 9010cf5b5d78fab6a785dadcb292eeeb1c82ab1b Mon Sep 17 00:00:00 2001 From: Brennan Magee Date: Thu, 21 Sep 2023 11:16:47 +0100 Subject: [PATCH 07/14] move celanup into correct place --- Jenkinsfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index adf746ac..049376d9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,6 +155,11 @@ pipeline { } } } + post { + cleanup { + xcoreCleanSandbox() + } + } } stage('Build Docs') { agent { label "docker" } @@ -178,9 +183,4 @@ pipeline { } } } - post { - cleanup { - xcoreCleanSandbox() - } - } } From 7571df29b41eb01c8044c0df775c071ce052e972 Mon Sep 17 00:00:00 2001 From: Shuchita Khare Date: Thu, 28 Sep 2023 14:29:01 +0100 Subject: [PATCH 08/14] mkdir -p --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 84702fd4..e3574485 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -124,7 +124,7 @@ pipeline { dir("${REPO}") { withTools(params.TOOLS_VERSION) { withVenv { - sh 'mkdir build' + sh 'mkdir -p build' dir("build") { sh 'rm CMakeCache.txt' sh 'cmake --toolchain ../xmos_cmake_toolchain/xs2a.cmake ..' From bf67b6f270c9313f2ddbc786df1489851282a8f4 Mon Sep 17 00:00:00 2001 From: Shuchita Khare Date: Thu, 28 Sep 2023 15:44:37 +0100 Subject: [PATCH 09/14] Add SNR plots to documentation --- Jenkinsfile | 243 ++++++++++++++++++++++++---------------------------- index.rst | 1 + 2 files changed, 115 insertions(+), 129 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e3574485..35a29caa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,6 +23,7 @@ pipeline { VIEW = getViewName(REPO) PYTHON_VERSION = "3.10.5" VENV_DIRNAME = ".venv" + XMOSDOC_VERSION = "pr-67" } options { skipDefaultCheckout() @@ -36,167 +37,151 @@ pipeline { ) } stages { - stage('Build and Docs') { - parallel { - stage('Build and Test') { - when { - expression { !env.GH_LABEL_DOC_ONLY.toBoolean() } - } - agent { - label 'x86_64&&macOS' // These agents have 24 cores so good for parallel xsim runs + stage('Build and Test') { + when { + expression { !env.GH_LABEL_DOC_ONLY.toBoolean() } + } + agent { + label 'x86_64&&docker' // These agents have 24 cores so good for parallel xsim runs + } + stages { + stage('Get repo') { + steps { + sh "mkdir ${REPO}" + // source checks require the directory + // name to be the same as the repo name + dir("${REPO}") { + // checkout repo + checkout scm + sh 'git submodule update --init --recursive --depth 1' + } } - stages { - stage('Get repo') { - steps { - sh "mkdir ${REPO}" - // source checks require the directory - // name to be the same as the repo name - dir("${REPO}") { - // checkout repo - checkout scm - sh 'git submodule update --init --recursive --depth 1' - } + } + stage ("Create Python environment") { + steps { + dir("${REPO}") { + createVenv('requirements.txt') + withVenv { + sh 'pip install -r requirements.txt' } } - stage ("Create Python environment") { - steps { - dir("${REPO}") { - createVenv('requirements.txt') - withVenv { - sh 'pip install -r requirements.txt' + } + } + stage('Library checks') { + steps { + dir("${REPO}") { + sh 'git clone git@github.com:xmos/infr_apps.git' + sh 'git clone git@github.com:xmos/infr_scripts_py.git' + // These are needed for xmake legacy build and also changelog check + sh 'git clone git@github.com:xmos/lib_logging.git' + sh 'git clone git@github.com:xmos/lib_xassert.git' + withVenv { + sh 'pip install -e infr_scripts_py' + sh 'pip install -e infr_apps' + dir("tests") { + withEnv(["XMOS_ROOT=.."]) { + localRunPytest('-s test_lib_checks.py -vv') } } } } - stage('Library checks') { - steps { - dir("${REPO}") { - sh 'git clone git@github.com:xmos/infr_apps.git' - sh 'git clone git@github.com:xmos/infr_scripts_py.git' - // These are needed for xmake legacy build and also changelog check - sh 'git clone git@github.com:xmos/lib_logging.git' - sh 'git clone git@github.com:xmos/lib_xassert.git' - withVenv { - sh 'pip install -e infr_scripts_py' - sh 'pip install -e infr_apps' - dir("tests") { - withEnv(["XMOS_ROOT=.."]) { - localRunPytest('-s test_lib_checks.py -vv') - } - } + } + } + stage('Test xmake build') { + steps { + runningOn(env.NODE_NAME) + dir("${REPO}") { + withTools(params.TOOLS_VERSION) { + withVenv { + dir("tests") { + localRunPytest('-k "legacy" -vv') } } } } - stage('Test xmake build') { - steps { - runningOn(env.NODE_NAME) - dir("${REPO}") { - withTools(params.TOOLS_VERSION) { - withVenv { - dir("tests") { - localRunPytest('-k "legacy" -vv') - } - } + } + } + stage('Run doc python') { + steps { + runningOn(env.NODE_NAME) + dir("${REPO}") { + withTools(params.TOOLS_VERSION) { + withVenv { + dir("doc/python") { + sh "python -m doc_asrc.py" + sh "docker pull ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION" + sh """docker run -u "\$(id -u):\$(id -g)" \ + --rm \ + -v ${WORKSPACE}:/build \ + ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" + archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true + sh "zip -r snr_build.zip _build" + archiveArtifacts artifacts: "snr_build.zip" } } } } - stage('Run doc python') { - steps { - runningOn(env.NODE_NAME) - dir("${REPO}") { - withTools(params.TOOLS_VERSION) { - withVenv { - dir("doc/python") { - sh "python -m doc_asrc.py" - sh "zip -r snr_build.zip _build" - archiveArtifacts artifacts: "snr_build.zip" - } - } + } + } + stage('Tests XS2') { + steps { + runningOn(env.NODE_NAME) + dir("${REPO}") { + withTools(params.TOOLS_VERSION) { + withVenv { + sh 'mkdir -p build' + dir("build") { + sh 'rm CMakeCache.txt' + sh 'cmake --toolchain ../xmos_cmake_toolchain/xs2a.cmake ..' + sh 'make test_ds3_voice test_us3_voice test_unity_gain_voice -j' } - } - } - } - stage('Tests XS2') { - steps { - runningOn(env.NODE_NAME) - dir("${REPO}") { - withTools(params.TOOLS_VERSION) { - withVenv { - sh 'mkdir -p build' - dir("build") { - sh 'rm CMakeCache.txt' - sh 'cmake --toolchain ../xmos_cmake_toolchain/xs2a.cmake ..' - sh 'make test_ds3_voice test_us3_voice test_unity_gain_voice -j' - } - dir("tests") { - localRunPytest('-n auto -k "xs2" -vv') - } - dir("build") { - sh 'rm CMakeCache.txt' // Cleanup XS2 cmake cache for next stage - } - } + dir("tests") { + localRunPytest('-n auto -k "xs2" -vv') + } + dir("build") { + sh 'rm CMakeCache.txt' // Cleanup XS2 cmake cache for next stage } } } } - stage('Tests XS3') { - steps { - runningOn(env.NODE_NAME) - dir("${REPO}") { - withTools(params.TOOLS_VERSION) { - withVenv { - dir("tests") { - localRunPytest('-m prepare') // Do all pre work like building and generating golden ref where needed + } + } + stage('Tests XS3') { + steps { + runningOn(env.NODE_NAME) + dir("${REPO}") { + withTools(params.TOOLS_VERSION) { + withVenv { + dir("tests") { + localRunPytest('-m prepare') // Do all pre work like building and generating golden ref where needed - // FF3 HiFi tests for OS3 and DS3 - localRunPytest('-m main -n auto -k "hifi_ff3" -vv') + // FF3 HiFi tests for OS3 and DS3 + localRunPytest('-m main -n auto -k "hifi_ff3" -vv') - // ASRC and SSRC tests across all in/out freqs and deviations (asrc only) - localRunPytest('-m main -n auto -k "mrhf" -vv') - archiveArtifacts artifacts: "mips_report*.csv", allowEmptyArchive: true + // ASRC and SSRC tests across all in/out freqs and deviations (asrc only) + localRunPytest('-m main -n auto -k "mrhf" -vv') + archiveArtifacts artifacts: "mips_report*.csv", allowEmptyArchive: true - // VPU enabled ff3 and rat tests - localRunPytest('-m main -k "vpu" -vv') // xdist not working yet so no -n auto + // VPU enabled ff3 and rat tests + localRunPytest('-m main -k "vpu" -vv') // xdist not working yet so no -n auto - // Profile the ASRC - localRunPytest('-m main -k "profile_asrc" -vv') - sh 'tree' - archiveArtifacts artifacts: "gprof_results/*.png", allowEmptyArchive: true - } - } + // Profile the ASRC + localRunPytest('-m main -k "profile_asrc" -vv') + sh 'tree' + archiveArtifacts artifacts: "gprof_results/*.png", allowEmptyArchive: true } } } } } - post { - cleanup { - xcoreCleanSandbox() - } - } } - stage('Build Docs') { - agent { label "docker" } - environment { XMOSDOC_VERSION = "pr-67" } - steps { - checkout scm - sh 'git submodule update --init --recursive --depth 1' - sh "docker pull ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION" - sh """docker run -u "\$(id -u):\$(id -g)" \ - --rm \ - -v ${WORKSPACE}:/build \ - ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" - archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true - } - post { - cleanup { - xcoreCleanSandbox() - } - } + } + post { + cleanup { + xcoreCleanSandbox() } } } + } } diff --git a/index.rst b/index.rst index 7497baa9..6d3b4a84 100644 --- a/index.rst +++ b/index.rst @@ -7,3 +7,4 @@ SAMPLE RATE CONVERSION :maxdepth: 5 ./doc/programming_guide/index.rst + ./doc/python/_build/rst/allPlots.rst From 05eb44e84f89591a767e979665d79709697f2102 Mon Sep 17 00:00:00 2001 From: Shuchita Khare Date: Thu, 28 Sep 2023 16:28:59 +0100 Subject: [PATCH 10/14] fix path --- Jenkinsfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 35a29caa..72f100c0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,15 +109,18 @@ pipeline { withVenv { dir("doc/python") { sh "python -m doc_asrc.py" - sh "docker pull ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION" - sh """docker run -u "\$(id -u):\$(id -g)" \ - --rm \ - -v ${WORKSPACE}:/build \ - ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" - archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true sh "zip -r snr_build.zip _build" archiveArtifacts artifacts: "snr_build.zip" } + sh "docker pull ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION" + sh """docker run -u "\$(id -u):\$(id -g)" \ + --rm \ + -v ${WORKSPACE}/${REPO}:/build \ + ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" + archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true + + + } } } } From 4503181ea0e597bcaa2a7afd3540387c1ac3f72d Mon Sep 17 00:00:00 2001 From: Shuchita Khare Date: Thu, 28 Sep 2023 16:36:13 +0100 Subject: [PATCH 11/14] Remove extra bracket --- Jenkinsfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 72f100c0..34479a80 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -118,9 +118,6 @@ pipeline { -v ${WORKSPACE}/${REPO}:/build \ ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true - - - } } } } From 13bd57af5dbd4cb515a155aeec2333d61388941d Mon Sep 17 00:00:00 2001 From: Shuchita Khare Date: Thu, 28 Sep 2023 16:49:55 +0100 Subject: [PATCH 12/14] ignore lib_logging in documentation build --- doc/exclude_patterns.inc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/exclude_patterns.inc b/doc/exclude_patterns.inc index 599c2e6c..934f156d 100644 --- a/doc/exclude_patterns.inc +++ b/doc/exclude_patterns.inc @@ -7,7 +7,9 @@ build/* *.md **/*.md +lib_logging/** +lib_xassert/** # We need to ensure these are not processed twice which breaks figure numbering. They are included via index.rst. **/resource_usage_asrc.rst -**/resource_usage_ssrc.rst \ No newline at end of file +**/resource_usage_ssrc.rst From dfe1f185180e7732e8701cad0d1488aa8fc01d85 Mon Sep 17 00:00:00 2001 From: Shuchita Khare Date: Fri, 29 Sep 2023 09:53:13 +0100 Subject: [PATCH 13/14] zip the documentation before archiving --- Jenkinsfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 34479a80..2e9653fb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -109,15 +109,14 @@ pipeline { withVenv { dir("doc/python") { sh "python -m doc_asrc.py" - sh "zip -r snr_build.zip _build" - archiveArtifacts artifacts: "snr_build.zip" } sh "docker pull ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION" sh """docker run -u "\$(id -u):\$(id -g)" \ --rm \ -v ${WORKSPACE}/${REPO}:/build \ ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" - archiveArtifacts artifacts: "doc/_build/**", allowEmptyArchive: true + sh "zip -r doc/_build doc_build.zip" + archiveArtifacts artifacts: "doc_build.zip", allowEmptyArchive: true } } } From 9079847986d2c16f84a9a4d48fdef00e54bf06b2 Mon Sep 17 00:00:00 2001 From: Shuchita Khare Date: Fri, 29 Sep 2023 10:07:19 +0100 Subject: [PATCH 14/14] fix zip command --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e9653fb..bb71a1e3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -115,7 +115,7 @@ pipeline { --rm \ -v ${WORKSPACE}/${REPO}:/build \ ghcr.io/xmos/doc_builder:$XMOSDOC_VERSION -v""" - sh "zip -r doc/_build doc_build.zip" + sh "zip -r doc_build.zip doc/_build" archiveArtifacts artifacts: "doc_build.zip", allowEmptyArchive: true } }