From 060ec4bf3c1562f1406a8e5803b5e9a131cf44c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Wed, 19 Jan 2022 08:03:25 +0100 Subject: [PATCH 01/11] proposal on caching --- docs/proposals/caching.adoc | 69 +++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 docs/proposals/caching.adoc diff --git a/docs/proposals/caching.adoc b/docs/proposals/caching.adoc new file mode 100644 index 00000000..513d1d5a --- /dev/null +++ b/docs/proposals/caching.adoc @@ -0,0 +1,69 @@ += Proposal: Caching + +== Purpose + +This proposal aims to support caching of dependencies +(https://github.com/opendevstack/ods-pipeline/issues/147) in order to speed up build times. + +== Background + +Currently there is a single PVC per project which is shared by all build pipelines. +As a consequence only a single build at a time can run for a project. + +The PVC is mounted as the workspace in all build tasks. + +The `ods-start` task wipes the PVC at the beginning of each build so that no data persists between build. + +Each tekton task which gets started mounts the workspace PVC which can take quiet noticeable wait times. + +== Solution + +In the initial implementation instead of a single PVC per project one PVC per repo will be used. The PVC will be used both as a workspace for the build and also as a cache of data between builds. + +A build task run will have the following directories available: + +* A repo checkout dir. This is the current workspace directory although I propose to rename it to checkout directory as for some technologies the actual build will not happen here but in a caching directory if available. + +With sufficient caching space available: + +* A global cache directory where the parent directory is indicated by parameter `global-cache-parent`. Cleanup will only spare directories with in it so that build tasks must keep stuff in a subdirectory. For example in `$(global-cache-parent)/go-modules/`. + +* A pipeline cache directory as indicated by parameter `pipeline-cache-dir`. + +In the initial implementation the parameters will be set to `/.cache/` and `/.cache-p//` if space permits and be empty if there is not sufficient space available. + +While initially the caches will be on the same PVC as the checkout PVC, this should not be relied on, for example by using hard links between the checkout dir and the cache area. In the future we may want to keep different PVC for these areas. For example the global cache parent could be one day per project and perhaps even one per project for each build technology such as go, node. + +Also in a future version the `pipeline-cache-dir` may be on a different PVC as the checkout PVC for example to enable faster cleanup of the pristine checkout dir by simply recreating it or not even having it on a PVC. + +Files in the cache locations would persist between builds unless they are removed by the cleanup mechanism. + +The build scripts are adjusted to take advantage of the cache parameters above. In addition each build task will support a new parameters `global-caching-mb` and `pipeline-caching-mb`. These defaults to 0 to indicate the task is not using caching. Otherwise it specify the minimum required cache space in MB. + +All build scripts are adjusted to log + +- the time longer running commands take. + +Furthermore the available disks pace is be logged as well (not sure where this should be done at the moment) + +=== ods-start and cleanup + +Build tasks can rely on that cache cleanup does not occur while they are running as well as that these would not be partially cleaned up. + +On the other hand build Tasks cannot rely on that a cache remains populated from a previous build. + +The cleanup strategy described here is subject to change even in patch versions and specific details must not be relied on. + +Pipeline cleanup happens during `ods-start`, so that no other build can run concurrently. + +Before cache cleanup `ods-start` cleans up the prior PVC checkout dir, which means everything except `/.cache/` or `/.cache-p/`. In particular to prevent having files at the top levels regular files or links below `/.cache/` or `/.cache-p/` are deleted. + +`ods-start` then cleans up separately the global and pipeline cache to fulfill the declared requirements of all build tasks in the odl.yaml. If disk space remains too low in a particular cache area a warning message is logged and the respective parameter `global-cache-parent` and/or `pipeline-cache-dir` will be empty. + + +== Pro + +TODO + +== Con + +TODO \ No newline at end of file From 565cd71e4e3b6fbc9c7eda74341e5a30acfba55d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Wed, 19 Jan 2022 09:06:04 +0100 Subject: [PATCH 02/11] add caching pros and cons --- docs/proposals/caching.adoc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/proposals/caching.adoc b/docs/proposals/caching.adoc index 513d1d5a..abfefece 100644 --- a/docs/proposals/caching.adoc +++ b/docs/proposals/caching.adoc @@ -62,8 +62,16 @@ Before cache cleanup `ods-start` cleans up the prior PVC checkout dir, which mea == Pro -TODO +* Caching enabled for tools using global cache + +* Caching enabled where it is best to build in a cached context per pipeline + +* Opt-in to caching can be used to document best practices for PVC sizing. + +* By having cleanup done in ods-start no need to have an interleaved cleanup process. == Con -TODO \ No newline at end of file +* Tools such as for typescript https://github.com/pnpm/pnpm[pnpm] and https://github.com/yarnpkg/berry[yarn berry] support deduplication allowing to reference artifacts from a central location without packaging them again. pnpn uses hard links to make this efficient. However hard linking is not supported across PVCs and thus could not be used with the proposed design as is. See also related discussion at https://github.com/opendevstack/ods-pipeline/discussions/411[node typescript security and caching - npm or yarn? #411]. + +* In some technologies caching is not mature and there is a desire to have caching be disabled for particular builds such as develop branch builds. At the moment this information is not readily configurable as proposed. Suggestions to fix this are welcome! \ No newline at end of file From f66096a7c238bb92c7c5e2480e6ac6e318197981 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Thu, 20 Jan 2022 07:08:23 +0100 Subject: [PATCH 03/11] build-typescript draft changes --- build/package/scripts/build-typescript.sh | 71 ++++++++++++++++++++--- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/build/package/scripts/build-typescript.sh b/build/package/scripts/build-typescript.sh index 87f991b1..ca357b91 100755 --- a/build/package/scripts/build-typescript.sh +++ b/build/package/scripts/build-typescript.sh @@ -22,6 +22,7 @@ copyLintReport() { BUILD_DIR="dist" OUTPUT_DIR="docker" WORKING_DIR="." +PIPELINE_CACHE_DIR="" ARTIFACT_PREFIX="" DEBUG="${DEBUG:-false}" MAX_LINT_WARNINGS="0" @@ -34,6 +35,9 @@ while [[ "$#" -gt 0 ]]; do --working-dir) WORKING_DIR="$2"; shift;; --working-dir=*) WORKING_DIR="${1#*=}";; + --pipeline-cache-dir) PIPELINE_CACHE_DIR="$2"; shift;; + --pipeline-cache-dir=*) PIPELINE_CACHE_DIR="${1#*=}";; + --output-dir) OUTPUT_DIR="$2"; shift;; --output-dir=*) OUTPUT_DIR="${1#*=}";; @@ -60,9 +64,23 @@ if [ "${DEBUG}" == "true" ]; then fi ROOT_DIR=$(pwd) -if [ "${WORKING_DIR}" != "." ]; then - cd "${WORKING_DIR}" - ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" +# NOTE: git is currently not available in build script +git_ref=$(cat .ods/git-ref) +case $git_ref in + master|develop) + echo "INFO: not using cache in master or develop branch" + PIPELINE_CACHE_DIR="" + ;; + *);; +esac + + +if [ -d "$PIPELINE_CACHE_DIR" ]; then + # NOTE: rsync is currently not available in build script + echo "Updating cache dir with sources ..." + # rsync -auvhp --itemize-changes --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" + # rsync -auvhp --progress --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" + rsync -auhp --info=progress2 --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" # the info progress2 does not work with verbose, needs rather new rsync version. fi echo "Configuring npm to use Nexus ..." @@ -78,14 +96,41 @@ if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWOR npm config set strict-ssl=false fi; +build_dir="${WORKING_DIR}" +output_dir_abs="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" +if [ -d "$PIPELINE_CACHE_DIR" ]; then + build_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" +fi + +if [ "${WORKING_DIR}" != "." ]; then + ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" +fi +if [ "${build_dir}" != "." ]; then + cd "${build_dir}" +fi + echo "Installing dependencies ..." +<<<<<<< HEAD npm ci --ignore-scripts +======= +start_time=$SECONDS +if [ -d "$PIPELINE_CACHE_DIR" ]; then + npm i +else + npm ci +fi +elapsed=$(( SECONDS - start_time )) +echo "Installing dependencies took $elapsed seconds" +>>>>>>> 4dc6da42 (build-typescript draft changes) echo "Linting ..." +start_time=$SECONDS set +e npx eslint src --ext "${LINT_FILE_EXT}" --format compact --max-warnings "${MAX_LINT_WARNINGS}" > eslint-report.txt exitcode=$? set -e +elapsed=$(( SECONDS - start_time )) +echo "linting took $elapsed seconds" if [ $exitcode == 0 ]; then echo "OK" > eslint-report.txt @@ -96,16 +141,25 @@ else fi echo "Building ..." +start_time=$SECONDS npm run build -mkdir -p "${OUTPUT_DIR}" -cp -r "${BUILD_DIR}" "${OUTPUT_DIR}/dist" +elapsed=$(( SECONDS - start_time )) +echo "build took $elapsed seconds" +mkdir -p "${output_dir_abs}" +cp -rv "${BUILD_DIR}" "${output_dir_abs}/dist" if [ "${COPY_NODE_MODULES}" = true ]; then - echo "Copying node_modules to ${OUTPUT_DIR}/dist/node_modules ..." - cp -r node_modules "${OUTPUT_DIR}/dist/node_modules" + echo "Copying node_modules to ${output_dir_abs}/dist/node_modules ..." + start_time=$SECONDS + cp -r node_modules "${output_dir_abs}/dist/node_modules" + elapsed=$(( SECONDS - start_time )) + echo "copying node_modules took $elapsed seconds" fi echo "Testing ..." +# Implement skipping tests for TypeScript #238 +# TODO: is this still needed? +# https://github.com/opendevstack/ods-pipeline/issues/238 if [ -f "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" ]; then echo "Test artifacts already present, skipping tests ..." # Copy artifacts to working directory so that the SonarQube scanner can pick them up later. @@ -131,4 +185,7 @@ else cp build/coverage/lcov.info "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" fi +# Provide disk usage so that one can use this to estimate needs + + supply-sonar-project-properties-default From 186142e2dd7b4906427a0ef4658785885f8232d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Thu, 20 Jan 2022 09:04:49 +0100 Subject: [PATCH 04/11] build-typescript draft changes enable build reuse via working-dir-commit-sha --- build/package/scripts/build-typescript.sh | 130 +++++++++++++++------- 1 file changed, 87 insertions(+), 43 deletions(-) diff --git a/build/package/scripts/build-typescript.sh b/build/package/scripts/build-typescript.sh index ca357b91..ed3da3a0 100755 --- a/build/package/scripts/build-typescript.sh +++ b/build/package/scripts/build-typescript.sh @@ -19,9 +19,48 @@ copyLintReport() { cp eslint-report.txt "${ROOT_DIR}/.ods/artifacts/lint-reports/${ARTIFACT_PREFIX}report.txt" } +copyTestReports() { + mkdir -p "${ROOT_DIR}/.ods/artifacts/xunit-reports" + cat build/test-results/test/report.xml + cp build/test-results/test/report.xml "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" + + mkdir -p "${ROOT_DIR}/.ods/artifacts/code-coverage" + cat build/coverage/clover.xml + cp build/coverage/clover.xml "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}clover.xml" + + cat build/coverage/coverage-final.json + cp build/coverage/coverage-final.json "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}coverage-final.json" + + cat build/coverage/lcov.info + cp build/coverage/lcov.info "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" +} + +copyBuildResults() { + mkdir -p "${output_dir_abs}" + cp -rv "${BUILD_DIR}" "${output_dir_abs}/dist" + + if [ "${COPY_NODE_MODULES}" = true ]; then + echo "Copying node_modules to ${output_dir_abs}/dist/node_modules ..." + start_time=$SECONDS + cp -r node_modules "${output_dir_abs}/dist/node_modules" + elapsed=$(( SECONDS - start_time )) + echo "copying node_modules took $elapsed seconds" + fi +} + +touch_dir_commit_hash() { + # this would enables cleanup based on these timestamps + # however this function should ideally be done outside of the individual + # build scripts. + mkdir -p "$build_dir/.ods/" + echo -n "$WORKING_DIR_COMMIT_SHA" > "$build_dir/.ods/git-dir-commit-sha" +} + + BUILD_DIR="dist" OUTPUT_DIR="docker" WORKING_DIR="." +WORKING_DIR_COMMIT_SHA="" PIPELINE_CACHE_DIR="" ARTIFACT_PREFIX="" DEBUG="${DEBUG:-false}" @@ -35,6 +74,11 @@ while [[ "$#" -gt 0 ]]; do --working-dir) WORKING_DIR="$2"; shift;; --working-dir=*) WORKING_DIR="${1#*=}";; + --working-dir-commit-sha) WORKING_DIR_COMMIT_SHA="$2"; shift;; + --working-dir-commit-sha=*) WORKING_DIR_COMMIT_SHA="${1#*=}";; + + # build pipeline should support skipping for example via commit tag + # in this case --pipeline-cache-dir would be omitted or set to an empty value. --pipeline-cache-dir) PIPELINE_CACHE_DIR="$2"; shift;; --pipeline-cache-dir=*) PIPELINE_CACHE_DIR="${1#*=}";; @@ -59,6 +103,10 @@ while [[ "$#" -gt 0 ]]; do *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done +if [ -z "$WORKING_DIR_COMMIT_SHA" ]; then + echo "--working-dir-commit-sha parameter is required."; exit 1 +fi + if [ "${DEBUG}" == "true" ]; then set -x fi @@ -74,15 +122,45 @@ case $git_ref in *);; esac +same_build_in_cache=false +build_dir="${WORKING_DIR}" +output_dir_abs="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" +if [ -n "$PIPELINE_CACHE_DIR" ]; then + build_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" + if [ ! -d "$build_dir" ]; then + mkdir -p "$build_dir" + fi + if [ -f "$build_dir/.ods/git-dir-commit-sha" ]; then + previous_dir_commit_sha=$(cat "$build_dir/.ods/git-dir-commit-sha") + if [ "$previous_dir_commit_sha" = "$WORKING_DIR_COMMIT_SHA" ]; then + same_build_in_cache=true + echo "INFO: build with same commit hash of dir $build_dir already in cache ($previous_dir_commit_sha)." + fi + fi + if [ "$same_build_in_cache" = "false" ]; then + # NOTE: rsync is currently not available in build script + echo "Updating cache dir with sources ..." + # rsync -auvhp --itemize-changes --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" + # rsync -auvhp --progress --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" + rsync -auhp --info=progress2 --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" # the info progress2 does not work with verbose, needs rather new rsync version. + fi +fi -if [ -d "$PIPELINE_CACHE_DIR" ]; then - # NOTE: rsync is currently not available in build script - echo "Updating cache dir with sources ..." - # rsync -auvhp --itemize-changes --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" - # rsync -auvhp --progress --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" - rsync -auhp --info=progress2 --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" # the info progress2 does not work with verbose, needs rather new rsync version. +if [ "${WORKING_DIR}" != "." ]; then + ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" +fi +if [ "${build_dir}" != "." ]; then + cd "${build_dir}" fi +if [ "$same_build_in_cache" = "true" ]; then + echo "Using prior build for same commit hash to copy build results and reports" + copyBuildResults + copyLintReport + copyTestReports + touch_dir_commit_hash + exit 0 +fi echo "Configuring npm to use Nexus ..." # Remove the protocol segment from NEXUS_URL NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g') @@ -96,19 +174,6 @@ if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWOR npm config set strict-ssl=false fi; -build_dir="${WORKING_DIR}" -output_dir_abs="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" -if [ -d "$PIPELINE_CACHE_DIR" ]; then - build_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" -fi - -if [ "${WORKING_DIR}" != "." ]; then - ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" -fi -if [ "${build_dir}" != "." ]; then - cd "${build_dir}" -fi - echo "Installing dependencies ..." <<<<<<< HEAD npm ci --ignore-scripts @@ -145,16 +210,7 @@ start_time=$SECONDS npm run build elapsed=$(( SECONDS - start_time )) echo "build took $elapsed seconds" -mkdir -p "${output_dir_abs}" -cp -rv "${BUILD_DIR}" "${output_dir_abs}/dist" - -if [ "${COPY_NODE_MODULES}" = true ]; then - echo "Copying node_modules to ${output_dir_abs}/dist/node_modules ..." - start_time=$SECONDS - cp -r node_modules "${output_dir_abs}/dist/node_modules" - elapsed=$(( SECONDS - start_time )) - echo "copying node_modules took $elapsed seconds" -fi +copyBuildResults echo "Testing ..." # Implement skipping tests for TypeScript #238 @@ -169,20 +225,8 @@ if [ -f "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" cp "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" lcov.info else npm run test - - mkdir -p "${ROOT_DIR}/.ods/artifacts/xunit-reports" - cat build/test-results/test/report.xml - cp build/test-results/test/report.xml "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" - - mkdir -p "${ROOT_DIR}/.ods/artifacts/code-coverage" - cat build/coverage/clover.xml - cp build/coverage/clover.xml "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}clover.xml" - - cat build/coverage/coverage-final.json - cp build/coverage/coverage-final.json "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}coverage-final.json" - - cat build/coverage/lcov.info - cp build/coverage/lcov.info "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" + copyTestReports + touch_dir_commit_hash fi # Provide disk usage so that one can use this to estimate needs From 80a0159afe9bfaed771c778f2a8d769d44ff2852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Mon, 24 Jan 2022 11:36:23 +0100 Subject: [PATCH 05/11] reduce build_dir confusion and report sizes --- build/package/scripts/build-typescript.sh | 46 ++++++++++++++--------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/build/package/scripts/build-typescript.sh b/build/package/scripts/build-typescript.sh index ed3da3a0..bbdcce93 100755 --- a/build/package/scripts/build-typescript.sh +++ b/build/package/scripts/build-typescript.sh @@ -52,8 +52,13 @@ touch_dir_commit_hash() { # this would enables cleanup based on these timestamps # however this function should ideally be done outside of the individual # build scripts. - mkdir -p "$build_dir/.ods/" - echo -n "$WORKING_DIR_COMMIT_SHA" > "$build_dir/.ods/git-dir-commit-sha" + mkdir -p "$build_root_dir/.ods/" + echo -n "$WORKING_DIR_COMMIT_SHA" > "$build_root_dir/.ods/git-dir-commit-sha" +} + +report_disk_usage() { + echo "Disk usage to estimate caching needs." + du -hs -- * } @@ -123,34 +128,42 @@ case $git_ref in esac same_build_in_cache=false -build_dir="${WORKING_DIR}" +build_root_dir="${WORKING_DIR}" output_dir_abs="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" if [ -n "$PIPELINE_CACHE_DIR" ]; then - build_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" - if [ ! -d "$build_dir" ]; then - mkdir -p "$build_dir" + build_root_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" + if [ ! -d "$build_root_dir" ]; then + mkdir -p "$build_root_dir" fi - if [ -f "$build_dir/.ods/git-dir-commit-sha" ]; then - previous_dir_commit_sha=$(cat "$build_dir/.ods/git-dir-commit-sha") + if [ -f "$build_root_dir/.ods/git-dir-commit-sha" ]; then + previous_dir_commit_sha=$(cat "$build_root_dir/.ods/git-dir-commit-sha") if [ "$previous_dir_commit_sha" = "$WORKING_DIR_COMMIT_SHA" ]; then same_build_in_cache=true - echo "INFO: build with same commit hash of dir $build_dir already in cache ($previous_dir_commit_sha)." + echo "INFO: build with same commit hash of dir $build_root_dir already in cache ($previous_dir_commit_sha)." fi fi if [ "$same_build_in_cache" = "false" ]; then # NOTE: rsync is currently not available in build script echo "Updating cache dir with sources ..." - # rsync -auvhp --itemize-changes --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" - # rsync -auvhp --progress --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" - rsync -auhp --info=progress2 --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" # the info progress2 does not work with verbose, needs rather new rsync version. + rsync -ah -v --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" + # -a = -rlptgoD + # -r = --recursive + # -l = --links recreate the symlink on the destination + # -p = --perms (details see man page): aims to set dest permissions same as source + # -t = --times tells rsync to transfer modification times + # -goD = --group --owner --devices --special + # -h = --hard-links reestablish hard links if they are both in the copied set + # -v = --verbose (multiple would be more verbose) + # copied content is shown by sending incremental file list. Which maybe empty. + # seems like this is all that is needed fi fi if [ "${WORKING_DIR}" != "." ]; then ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" fi -if [ "${build_dir}" != "." ]; then - cd "${build_dir}" +if [ "${build_root_dir}" != "." ]; then + cd "${build_root_dir}" fi if [ "$same_build_in_cache" = "true" ]; then @@ -158,7 +171,7 @@ if [ "$same_build_in_cache" = "true" ]; then copyBuildResults copyLintReport copyTestReports - touch_dir_commit_hash + report_disk_usage exit 0 fi echo "Configuring npm to use Nexus ..." @@ -229,7 +242,6 @@ else touch_dir_commit_hash fi -# Provide disk usage so that one can use this to estimate needs - +report_disk_usage supply-sonar-project-properties-default From cd9b90337bd60bd66c00f6f441ea0b8bf3dd864b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Thu, 27 Jan 2022 16:18:17 +0100 Subject: [PATCH 06/11] proposal evolution towards workspace cache --- build/package/scripts/build-typescript.sh | 103 ++++++---------- docs/proposals/caching.adoc | 137 ++++++++++++++++++---- 2 files changed, 146 insertions(+), 94 deletions(-) diff --git a/build/package/scripts/build-typescript.sh b/build/package/scripts/build-typescript.sh index bbdcce93..955a812a 100755 --- a/build/package/scripts/build-typescript.sh +++ b/build/package/scripts/build-typescript.sh @@ -36,24 +36,21 @@ copyTestReports() { } copyBuildResults() { - mkdir -p "${output_dir_abs}" - cp -rv "${BUILD_DIR}" "${output_dir_abs}/dist" + mkdir -p "${OUTPUT_DIR_ABS}" + cp -rv "${BUILD_DIR}" "${OUTPUT_DIR_ABS}/dist" if [ "${COPY_NODE_MODULES}" = true ]; then - echo "Copying node_modules to ${output_dir_abs}/dist/node_modules ..." + echo "Copying node_modules to ${OUTPUT_DIR_ABS}/dist/node_modules ..." start_time=$SECONDS - cp -r node_modules "${output_dir_abs}/dist/node_modules" + cp -r node_modules "${OUTPUT_DIR_ABS}/dist/node_modules" elapsed=$(( SECONDS - start_time )) echo "copying node_modules took $elapsed seconds" fi } -touch_dir_commit_hash() { - # this would enables cleanup based on these timestamps - # however this function should ideally be done outside of the individual - # build scripts. - mkdir -p "$build_root_dir/.ods/" - echo -n "$WORKING_DIR_COMMIT_SHA" > "$build_root_dir/.ods/git-dir-commit-sha" +save_built_commit_hash() { + mkdir -p ".ods/" + echo -n "$working_dir_commit_sha" > ".ods/git-dir-last-built-commit-sha" } report_disk_usage() { @@ -65,28 +62,25 @@ report_disk_usage() { BUILD_DIR="dist" OUTPUT_DIR="docker" WORKING_DIR="." -WORKING_DIR_COMMIT_SHA="" -PIPELINE_CACHE_DIR="" ARTIFACT_PREFIX="" DEBUG="${DEBUG:-false}" MAX_LINT_WARNINGS="0" LINT_FILE_EXT=".js,.ts,.jsx,.tsx,.svelte" COPY_NODE_MODULES="false" +# Uses $WORKING_DIR/.ods/git-dir-commit-sha to recognize that it is running in a cached workspace +# +# This script stores the current commit hash from the file above when +# the build completes under +# $WORKING_DIR/.ods/git-dir-last-built-commit-sha +# In a future build this file is used to determine whether this build can be skipped. + while [[ "$#" -gt 0 ]]; do case $1 in --working-dir) WORKING_DIR="$2"; shift;; --working-dir=*) WORKING_DIR="${1#*=}";; - --working-dir-commit-sha) WORKING_DIR_COMMIT_SHA="$2"; shift;; - --working-dir-commit-sha=*) WORKING_DIR_COMMIT_SHA="${1#*=}";; - - # build pipeline should support skipping for example via commit tag - # in this case --pipeline-cache-dir would be omitted or set to an empty value. - --pipeline-cache-dir) PIPELINE_CACHE_DIR="$2"; shift;; - --pipeline-cache-dir=*) PIPELINE_CACHE_DIR="${1#*=}";; - --output-dir) OUTPUT_DIR="$2"; shift;; --output-dir=*) OUTPUT_DIR="${1#*=}";; @@ -108,65 +102,36 @@ while [[ "$#" -gt 0 ]]; do *) echo "Unknown parameter passed: $1"; exit 1;; esac; shift; done -if [ -z "$WORKING_DIR_COMMIT_SHA" ]; then - echo "--working-dir-commit-sha parameter is required."; exit 1 -fi - if [ "${DEBUG}" == "true" ]; then set -x fi ROOT_DIR=$(pwd) -# NOTE: git is currently not available in build script -git_ref=$(cat .ods/git-ref) -case $git_ref in - master|develop) - echo "INFO: not using cache in master or develop branch" - PIPELINE_CACHE_DIR="" - ;; - *);; -esac - -same_build_in_cache=false -build_root_dir="${WORKING_DIR}" -output_dir_abs="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" -if [ -n "$PIPELINE_CACHE_DIR" ]; then - build_root_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" - if [ ! -d "$build_root_dir" ]; then - mkdir -p "$build_root_dir" - fi - if [ -f "$build_root_dir/.ods/git-dir-commit-sha" ]; then - previous_dir_commit_sha=$(cat "$build_root_dir/.ods/git-dir-commit-sha") - if [ "$previous_dir_commit_sha" = "$WORKING_DIR_COMMIT_SHA" ]; then - same_build_in_cache=true - echo "INFO: build with same commit hash of dir $build_root_dir already in cache ($previous_dir_commit_sha)." - fi - fi - if [ "$same_build_in_cache" = "false" ]; then - # NOTE: rsync is currently not available in build script - echo "Updating cache dir with sources ..." - rsync -ah -v --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" - # -a = -rlptgoD - # -r = --recursive - # -l = --links recreate the symlink on the destination - # -p = --perms (details see man page): aims to set dest permissions same as source - # -t = --times tells rsync to transfer modification times - # -goD = --group --owner --devices --special - # -h = --hard-links reestablish hard links if they are both in the copied set - # -v = --verbose (multiple would be more verbose) - # copied content is shown by sending incremental file list. Which maybe empty. - # seems like this is all that is needed - fi + +already_built=false +# The following +abs_build_dir="${ROOT_DIR}/${WORKING_DIR}" +OUTPUT_DIR_ABS="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" +if [ ! -f "$WORKING_DIR/.ods/git-dir-commit-sha" ]; then + echo "--working-dir-commit-sha parameter is required."; exit 1 +fi +working_dir_commit_sha=$(cat "$abs_build_dir/.ods/git-dir-commit-sha") +working_dir_last_built_commit_sha="" +[ -f "$abs_build_dir/.ods/git-dir-last-built-commit-sha" ] && working_dir_last_built_commit_sha=$(cat "$abs_build_dir/.ods/git-dir-last-built-commit-sha") + +if [ "$working_dir_last_built_commit_sha" = "$working_dir_commit_sha" ]; then + already_built=true + echo "INFO: build with same commit hash of dir $abs_build_dir was cached and is still in workspace ($working_dir_commit_sha)." fi if [ "${WORKING_DIR}" != "." ]; then ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" fi -if [ "${build_root_dir}" != "." ]; then - cd "${build_root_dir}" +if [ "${WORKING_DIR}" != "." ]; then + cd "${WORKING_DIR}" fi -if [ "$same_build_in_cache" = "true" ]; then +if [ "$already_built" = "true" ]; then echo "Using prior build for same commit hash to copy build results and reports" copyBuildResults copyLintReport @@ -192,7 +157,7 @@ echo "Installing dependencies ..." npm ci --ignore-scripts ======= start_time=$SECONDS -if [ -d "$PIPELINE_CACHE_DIR" ]; then +if [ -n "$working_dir_last_built_commit_sha" ]; then npm i else npm ci @@ -239,8 +204,8 @@ if [ -f "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" else npm run test copyTestReports - touch_dir_commit_hash fi +save_built_commit_hash report_disk_usage diff --git a/docs/proposals/caching.adoc b/docs/proposals/caching.adoc index abfefece..4e44ff16 100644 --- a/docs/proposals/caching.adoc +++ b/docs/proposals/caching.adoc @@ -8,70 +8,157 @@ This proposal aims to support caching of dependencies == Background Currently there is a single PVC per project which is shared by all build pipelines. -As a consequence only a single build at a time can run for a project. +As a consequence only a single build at a time should run for a project. At the moment due to #394 parallel builds can happen, but with #407 and #160 solved, it will be "one build at a time for a repository". The PVC is mounted as the workspace in all build tasks. -The `ods-start` task wipes the PVC at the beginning of each build so that no data persists between build. +The `ods-start` task wipes the PVC at the beginning of each build so that no data persists between builds. -Each tekton task which gets started mounts the workspace PVC which can take quiet noticeable wait times. +Each running tekton task mounts the workspace PVC which typically takes a noticeable time. As a consequence it makes sense to not introduce additional tasks. == Solution -In the initial implementation instead of a single PVC per project one PVC per repo will be used. The PVC will be used both as a workspace for the build and also as a cache of data between builds. +In the initial implementation instead of a single PVC per project one PVC per repo will be used. -A build task run will have the following directories available: +There are two possible ways to cache which are largely independent, except that they would store files on the same PVC: -* A repo checkout dir. This is the current workspace directory although I propose to rename it to checkout directory as for some technologies the actual build will not happen here but in a caching directory if available. +1. A workspace cache enables to rebuild using the prior builds workspace akin to local development. -With sufficient caching space available: +2. A global cache allows storing dependencies. This is primarily intended for languages which are not supported by Nexus or similar service running in the cluster. -* A global cache directory where the parent directory is indicated by parameter `global-cache-parent`. Cleanup will only spare directories with in it so that build tasks must keep stuff in a subdirectory. For example in `$(global-cache-parent)/go-modules/`. +=== Workspace cache -* A pipeline cache directory as indicated by parameter `pipeline-cache-dir`. +Build tasks run in a workspace directory. Workspace caching allows the workspace directory to persist from prior pipeline runs -- space on the PVC permitting. -In the initial implementation the parameters will be set to `/.cache/` and `/.cache-p//` if space permits and be empty if there is not sufficient space available. +.TODO figure out where/how to implement workspace cache disablement. -While initially the caches will be on the same PVC as the checkout PVC, this should not be relied on, for example by using hard links between the checkout dir and the cache area. In the future we may want to keep different PVC for these areas. For example the global cache parent could be one day per project and perhaps even one per project for each build technology such as go, node. + -Also in a future version the `pipeline-cache-dir` may be on a different PVC as the checkout PVC for example to enable faster cleanup of the pristine checkout dir by simply recreating it or not even having it on a PVC. +To ensure that mainline builds cannot be impacted by workspace caching ods-pipeline ensures the following: -Files in the cache locations would persist between builds unless they are removed by the cleanup mechanism. +- Build pipelines for simple branch names will disable workspace caching. +- When opening a pull request a non workspace cached build is triggered if the current build of this commit was workspace cashed (or always). +- Pushing commits when a pull request is already open will disable workspace caching. -The build scripts are adjusted to take advantage of the cache parameters above. In addition each build task will support a new parameters `global-caching-mb` and `pipeline-caching-mb`. These defaults to 0 to indicate the task is not using caching. Otherwise it specify the minimum required cache space in MB. +ods-pipeline should support disable build with a special tag in a commit comment. + +Furthermore only build tasks for which workspace caching provides a significant performance gain will support workspace caching. Candidates are node and python related build tasks. -All build scripts are adjusted to log +* The workspace directory is where the source code is checked out. This is the current working directory when a build task starts running. Without caching the workspace directory will contain the checked out sources. A task will build at the workspace directory or below if `WORKING_DIR` is not set to `"."`. -- the time longer running commands take. +With sufficient caching space available the workspace directory persists from a prior build if in `ods.yaml` workspace caching is enabled. By default workspace caching is not enabled. -Furthermore the available disks pace is be logged as well (not sure where this should be done at the moment) +Without caching enabled `ods-start` checks out sources via `git-init` unchanged from the current implementation. With caching and if the workspace already exists, then the sources are updated via `git pull` instead and other files will be left alone, so that the build can take advantage of already installed dependencies for example. + +A new pipeline task parameter is introduced: + +* `cache-workspace-require-space-mb` a number defaulting to `0`. This is the number of MB required for the cache workspace for this build task. If greater 0 caching will be enabled space permitting and if not disabled as described above. + +Task implementation are expected to not pass the parameter to the build scripts as these should not be affected by this. + +When `ods-start` determines that workspace caching is available, it adds the following file: + +* `$WORKING_DIR/.ods/git-dir-commit-sha` which contains the git commit sha of the working directory (no whitespace or newlines). Build tasks can use this to avoid rebuilding when there were no changes in their working dir. + +Build tasks supporting caching will also be adjusted to + +- Log how long build commands which may be long running take in seconds. + +- Log available disk space well. This should be implemented so that one can see what sized to require. + +=== Global cache + +Only build tasks for which a global cache provides a significant performance gain will support global caching. At the moment the primary candidates is the go language. Nexus does not support go and we have no similar go dependency artifacts manager in place. + +The following new parameter is introduced to build tasks supporting global caching: + +* `cache-global-require-space-mb` a number defaulting to `0`. This is the number of MB required for the global cache for this build task. If greater 0 caching will be enabled space permitting. + +In addition to these build task parameters, the build tasks also receives the following: + +* File `.ods/cache-global-parent-dir` contains an absolute path without trailing '/' to an existing directory (no whitespace or newlines). If this file does not exits the task must not use global caching as space might have run out and a prior cache may have been deleted. + +Cleanup will spare directories below the cache-global-parent-dir so that build tasks must keep their cached files in a subdirectory they create. For example in `$(cache-global-parent-dir)/go-modules/` where `go-modules` is called technology-name. + +The global cache and workspace directory will be on the same PVC to enable build technologies utilizing the cache without needing to copy files by hard linking. === ods-start and cleanup -Build tasks can rely on that cache cleanup does not occur while they are running as well as that these would not be partially cleaned up. +Build tasks supporting caching can count on: + +* cache cleanups not occurring while they are running +* cleanups will not be partially completed. -On the other hand build Tasks cannot rely on that a cache remains populated from a previous build. +On the other hand build tasks must not assume that the cache is still available from a prior build. The cleanup strategy described here is subject to change even in patch versions and specific details must not be relied on. -Pipeline cleanup happens during `ods-start`, so that no other build can run concurrently. +Pipeline cleanup happens during `ods-start` + +The following locations on the PVC are used: + +- `/ws/` for uncached workspaces. These are recognized if none of there build tasks enabled caching. +- `/.cache-ws//` for cached workspaces. +- `/.cache//` for global caches of a particular build technology. The technology-name would be defined by the build script. + +Before cache cleanup `ods-start` cleans up `/ws/` on the PVC. + +Next all files directly underneath `/.cache/` which are not directories are deleted. This prevents tasks to forget to define and use a technology-name. -Before cache cleanup `ods-start` cleans up the prior PVC checkout dir, which means everything except `/.cache/` or `/.cache-p/`. In particular to prevent having files at the top levels regular files or links below `/.cache/` or `/.cache-p/` are deleted. +**workspace cache cleanup** -`ods-start` then cleans up separately the global and pipeline cache to fulfill the declared requirements of all build tasks in the odl.yaml. If disk space remains too low in a particular cache area a warning message is logged and the respective parameter `global-cache-parent` and/or `pipeline-cache-dir` will be empty. +The workspace cache cleanup is skipped if the pipeline has workspace caching disabled to avoid merging workspace cached pipelines into a mainline as describe earlier. + +.TODO this could be done by creating the pipeline with `cache-workspace-require-space-mb` forcefully set to 0 or via another flag. + +`ods-start` then determines: + +- `workspace-required-space-mb-total` :== sum of all declared `cache-workspace-require-space-mb` of build tasks contained in `ods.yaml` + +If this is 0 the next step is to look at global cache cleanup some steps below. + +Otherwise `ods-start` will determine the size of the workspace directory with a variation of `du /.cache-ws//` and convert the result to megabytes. + +Until there is not enough free space: + +- Deleted the oldest workspace. Age should be available via `stat -f '%m' .git/FETCH_HEAD` (see https://stackoverflow.com/a/9229377). +- If there is none continue below. + +If there is still not enough free space: + +- delete `/.cache-ws/` if it exists +- continue the build in `/ws`, so that its `cwd` is at `/ws` and that files `$WORKING_DIR/.ods/git-dir-commit-sha` will not be created for each of the build tasks. + +**global cache cleanup** + +`ods-start` determines: + +- `cache-required-space-mb-total` :== sum of all declared `cache-global-require-space-mb` of build tasks contained in `ods.yaml` + +The global cache cleanup deletes in a similar way as the workspace cache cleaning just described but the cleanup candidates are folders at: `/.cache//`. + +If there is not sufficient space in the global cache after cleanup the file `.ods/cache-global-parent-dir` will not exist in the workspace directory. == Pro * Caching enabled for tools using global cache -* Caching enabled where it is best to build in a cached context per pipeline +* Workspace caching enables speedup for technologies which benefit most from an incremental build as opposed to a global dependency cache. + +* Multi build repos can be better supported by avoiding to rebuild when their working directory does not change. + +* Ensuring a mainline build will always be non workspace cached before merging ensures that (workspace) cache issues cannot impact the main build. * Opt-in to caching can be used to document best practices for PVC sizing. * By having cleanup done in ods-start no need to have an interleaved cleanup process. +* Having workspaces and caches on the same PVC enables using modern newer technologies such as https://github.com/pnpm/pnpm[pnpm] or https://github.com/yarnpkg/berry[yarn berry (?)] which support deduplication allowing to reference artifacts from a central location without packaging them again. pnpn uses hard links to make this efficient. + == Con -* Tools such as for typescript https://github.com/pnpm/pnpm[pnpm] and https://github.com/yarnpkg/berry[yarn berry] support deduplication allowing to reference artifacts from a central location without packaging them again. pnpn uses hard links to make this efficient. However hard linking is not supported across PVCs and thus could not be used with the proposed design as is. See also related discussion at https://github.com/opendevstack/ods-pipeline/discussions/411[node typescript security and caching - npm or yarn? #411]. +* Avoiding unnecessary rebuild of sub builds when their working dir did not change by using workspace caching increases the complexity of the build scripts. Perhaps there is a better way to achieve this as the reports are already available and the build artifacts could be stored in Nexus or even extracted from a prior image. + +* Workspace caching does not increase performance of FE builds much as there the bundling takes typically the the most time. Perhaps newer tools such as https://github.com/evanw/esbuild[esbuild] or https://github.com/yarnpkg/berry[yarn berry] would help to speed up build times without requiring workspace caching. + +* The implementation to ensure mainline builds will always have a non workspace cached build may have bugs and thus the build may still be afflicted by cache issues which might pop up much later. -* In some technologies caching is not mature and there is a desire to have caching be disabled for particular builds such as develop branch builds. At the moment this information is not readily configurable as proposed. Suggestions to fix this are welcome! \ No newline at end of file +* Workspace caching adds complexity (unless we find it can be reduced to an acceptable level) From 86389a803e4e1f27722d5abcbe525e8d978c0b28 Mon Sep 17 00:00:00 2001 From: henrjk Date: Tue, 8 Feb 2022 11:40:30 +0100 Subject: [PATCH 07/11] Focus on global caching only This also removes changes intended showcase workspace caching for typescript. --- build/package/scripts/build-typescript.sh | 152 +++++++++------------- docs/proposals/caching.adoc | 138 ++++++-------------- 2 files changed, 99 insertions(+), 191 deletions(-) diff --git a/build/package/scripts/build-typescript.sh b/build/package/scripts/build-typescript.sh index 955a812a..e462aca6 100755 --- a/build/package/scripts/build-typescript.sh +++ b/build/package/scripts/build-typescript.sh @@ -19,68 +19,25 @@ copyLintReport() { cp eslint-report.txt "${ROOT_DIR}/.ods/artifacts/lint-reports/${ARTIFACT_PREFIX}report.txt" } -copyTestReports() { - mkdir -p "${ROOT_DIR}/.ods/artifacts/xunit-reports" - cat build/test-results/test/report.xml - cp build/test-results/test/report.xml "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" - - mkdir -p "${ROOT_DIR}/.ods/artifacts/code-coverage" - cat build/coverage/clover.xml - cp build/coverage/clover.xml "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}clover.xml" - - cat build/coverage/coverage-final.json - cp build/coverage/coverage-final.json "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}coverage-final.json" - - cat build/coverage/lcov.info - cp build/coverage/lcov.info "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" -} - -copyBuildResults() { - mkdir -p "${OUTPUT_DIR_ABS}" - cp -rv "${BUILD_DIR}" "${OUTPUT_DIR_ABS}/dist" - - if [ "${COPY_NODE_MODULES}" = true ]; then - echo "Copying node_modules to ${OUTPUT_DIR_ABS}/dist/node_modules ..." - start_time=$SECONDS - cp -r node_modules "${OUTPUT_DIR_ABS}/dist/node_modules" - elapsed=$(( SECONDS - start_time )) - echo "copying node_modules took $elapsed seconds" - fi -} - -save_built_commit_hash() { - mkdir -p ".ods/" - echo -n "$working_dir_commit_sha" > ".ods/git-dir-last-built-commit-sha" -} - -report_disk_usage() { - echo "Disk usage to estimate caching needs." - du -hs -- * -} - - BUILD_DIR="dist" OUTPUT_DIR="docker" WORKING_DIR="." +PIPELINE_CACHE_DIR="" ARTIFACT_PREFIX="" DEBUG="${DEBUG:-false}" MAX_LINT_WARNINGS="0" LINT_FILE_EXT=".js,.ts,.jsx,.tsx,.svelte" COPY_NODE_MODULES="false" -# Uses $WORKING_DIR/.ods/git-dir-commit-sha to recognize that it is running in a cached workspace -# -# This script stores the current commit hash from the file above when -# the build completes under -# $WORKING_DIR/.ods/git-dir-last-built-commit-sha -# In a future build this file is used to determine whether this build can be skipped. - while [[ "$#" -gt 0 ]]; do case $1 in --working-dir) WORKING_DIR="$2"; shift;; --working-dir=*) WORKING_DIR="${1#*=}";; + --pipeline-cache-dir) PIPELINE_CACHE_DIR="$2"; shift;; + --pipeline-cache-dir=*) PIPELINE_CACHE_DIR="${1#*=}";; + --output-dir) OUTPUT_DIR="$2"; shift;; --output-dir=*) OUTPUT_DIR="${1#*=}";; @@ -107,38 +64,25 @@ if [ "${DEBUG}" == "true" ]; then fi ROOT_DIR=$(pwd) - -already_built=false -# The following -abs_build_dir="${ROOT_DIR}/${WORKING_DIR}" -OUTPUT_DIR_ABS="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" -if [ ! -f "$WORKING_DIR/.ods/git-dir-commit-sha" ]; then - echo "--working-dir-commit-sha parameter is required."; exit 1 -fi -working_dir_commit_sha=$(cat "$abs_build_dir/.ods/git-dir-commit-sha") -working_dir_last_built_commit_sha="" -[ -f "$abs_build_dir/.ods/git-dir-last-built-commit-sha" ] && working_dir_last_built_commit_sha=$(cat "$abs_build_dir/.ods/git-dir-last-built-commit-sha") - -if [ "$working_dir_last_built_commit_sha" = "$working_dir_commit_sha" ]; then - already_built=true - echo "INFO: build with same commit hash of dir $abs_build_dir was cached and is still in workspace ($working_dir_commit_sha)." -fi - -if [ "${WORKING_DIR}" != "." ]; then - ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" -fi -if [ "${WORKING_DIR}" != "." ]; then - cd "${WORKING_DIR}" +# NOTE: git is currently not available in build script +git_ref=$(cat .ods/git-ref) +case $git_ref in + master|develop) + echo "INFO: not using cache in master or develop branch" + PIPELINE_CACHE_DIR="" + ;; + *);; +esac + + +if [ -d "$PIPELINE_CACHE_DIR" ]; then + # NOTE: rsync is currently not available in build script + echo "Updating cache dir with sources ..." + # rsync -auvhp --itemize-changes --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" + # rsync -auvhp --progress --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" + rsync -auhp --info=progress2 --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" # the info progress2 does not work with verbose, needs rather new rsync version. fi -if [ "$already_built" = "true" ]; then - echo "Using prior build for same commit hash to copy build results and reports" - copyBuildResults - copyLintReport - copyTestReports - report_disk_usage - exit 0 -fi echo "Configuring npm to use Nexus ..." # Remove the protocol segment from NEXUS_URL NEXUS_HOST=$(echo "${NEXUS_URL}" | sed -E 's/^\s*.*:\/\///g') @@ -152,19 +96,21 @@ if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWOR npm config set strict-ssl=false fi; +build_dir="${WORKING_DIR}" +output_dir_abs="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" +if [ -d "$PIPELINE_CACHE_DIR" ]; then + build_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" +fi + +if [ "${WORKING_DIR}" != "." ]; then + ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" +fi +if [ "${build_dir}" != "." ]; then + cd "${build_dir}" +fi + echo "Installing dependencies ..." -<<<<<<< HEAD npm ci --ignore-scripts -======= -start_time=$SECONDS -if [ -n "$working_dir_last_built_commit_sha" ]; then - npm i -else - npm ci -fi -elapsed=$(( SECONDS - start_time )) -echo "Installing dependencies took $elapsed seconds" ->>>>>>> 4dc6da42 (build-typescript draft changes) echo "Linting ..." start_time=$SECONDS @@ -188,7 +134,16 @@ start_time=$SECONDS npm run build elapsed=$(( SECONDS - start_time )) echo "build took $elapsed seconds" -copyBuildResults +mkdir -p "${output_dir_abs}" +cp -rv "${BUILD_DIR}" "${output_dir_abs}/dist" + +if [ "${COPY_NODE_MODULES}" = true ]; then + echo "Copying node_modules to ${output_dir_abs}/dist/node_modules ..." + start_time=$SECONDS + cp -r node_modules "${output_dir_abs}/dist/node_modules" + elapsed=$(( SECONDS - start_time )) + echo "copying node_modules took $elapsed seconds" +fi echo "Testing ..." # Implement skipping tests for TypeScript #238 @@ -203,10 +158,23 @@ if [ -f "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" cp "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" lcov.info else npm run test - copyTestReports + + mkdir -p "${ROOT_DIR}/.ods/artifacts/xunit-reports" + cat build/test-results/test/report.xml + cp build/test-results/test/report.xml "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" + + mkdir -p "${ROOT_DIR}/.ods/artifacts/code-coverage" + cat build/coverage/clover.xml + cp build/coverage/clover.xml "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}clover.xml" + + cat build/coverage/coverage-final.json + cp build/coverage/coverage-final.json "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}coverage-final.json" + + cat build/coverage/lcov.info + cp build/coverage/lcov.info "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" fi -save_built_commit_hash -report_disk_usage +# Provide disk usage so that one can use this to estimate needs + supply-sonar-project-properties-default diff --git a/docs/proposals/caching.adoc b/docs/proposals/caching.adoc index 4e44ff16..ac881501 100644 --- a/docs/proposals/caching.adoc +++ b/docs/proposals/caching.adoc @@ -5,6 +5,12 @@ This proposal aims to support caching of dependencies (https://github.com/opendevstack/ods-pipeline/issues/147) in order to speed up build times. +An earlier version had proposed also a workspace caching scheme. +While workspace caching can provide a large benefit for some languages such as +Python and Node it also introduces complexities and could hide dependency issues. +As a consequence this proposal only focuses on a global cache. +A workspace caching proposal could be made at a later time. + == Background Currently there is a single PVC per project which is shared by all build pipelines. @@ -20,65 +26,34 @@ Each running tekton task mounts the workspace PVC which typically takes a notice In the initial implementation instead of a single PVC per project one PVC per repo will be used. -There are two possible ways to cache which are largely independent, except that they would store files on the same PVC: - -1. A workspace cache enables to rebuild using the prior builds workspace akin to local development. - -2. A global cache allows storing dependencies. This is primarily intended for languages which are not supported by Nexus or similar service running in the cluster. - -=== Workspace cache - -Build tasks run in a workspace directory. Workspace caching allows the workspace directory to persist from prior pipeline runs -- space on the PVC permitting. +The same PVC is used for the workspace (where a repo is checked out in) and the global cache. -.TODO figure out where/how to implement workspace cache disablement. +The global cache enables build tasks to store cached files for at least the following purposes: -To ensure that mainline builds cannot be impacted by workspace caching ods-pipeline ensures the following: +1. Dependency caching. Languages that use dependencies as packaged artifacts in Nexus may not benefit much from caching dependencies. Unless there is a clear indication that there is a large performance win, languages should not use the global cache if they can use Nexus. A language that currently is not supported by Nexus is go. Another example where a large performance win could be expected are `pnpm` or `yarn` which can install dependencies by referencing the artifacts on the same file system in a very efficient way. In other words there is no need to unpack/copy the dependencies from a global file system cache. -- Build pipelines for simple branch names will disable workspace caching. -- When opening a pull request a non workspace cached build is triggered if the current build of this commit was workspace cashed (or always). -- Pushing commits when a pull request is already open will disable workspace caching. +2. Cache prior build output to support build skipping (see proposal at https://github.com/opendevstack/ods-pipeline/pull/423). Usage of the global cache for this purpose should work analog to how this is done for dependency caching but it is not further elaborated on here as this is covered in the mentioned proposal. -ods-pipeline should support disable build with a special tag in a commit comment. - -Furthermore only build tasks for which workspace caching provides a significant performance gain will support workspace caching. Candidates are node and python related build tasks. - -* The workspace directory is where the source code is checked out. This is the current working directory when a build task starts running. Without caching the workspace directory will contain the checked out sources. A task will build at the workspace directory or below if `WORKING_DIR` is not set to `"."`. - -With sufficient caching space available the workspace directory persists from a prior build if in `ods.yaml` workspace caching is enabled. By default workspace caching is not enabled. +3. There may be other usages for example for the gradle wrapper that could perhaps benefit from a global cache. -Without caching enabled `ods-start` checks out sources via `git-init` unchanged from the current implementation. With caching and if the workspace already exists, then the sources are updated via `git pull` instead and other files will be left alone, so that the build can take advantage of already installed dependencies for example. +The following new parameter is introduced to build tasks supporting dependency caching: -A new pipeline task parameter is introduced: +* `cache-deps` a boolean defaulting to `false`. Only if set to `true` the task will used dependency caching. -* `cache-workspace-require-space-mb` a number defaulting to `0`. This is the number of MB required for the cache workspace for this build task. If greater 0 caching will be enabled space permitting and if not disabled as described above. +In addition to the build task parameter, build tasks also receives the following: -Task implementation are expected to not pass the parameter to the build scripts as these should not be affected by this. +* File `.ods/cache-deps-parent-dir` contains an absolute path without trailing '/' to an existing directory (no whitespace or newlines). If file `.ods/cache-deps-parent-dir` does not exits the task must not use dependency caching. This is used to switch off dependency caching dynamically for example with a special tag in a git commit message. -When `ods-start` determines that workspace caching is available, it adds the following file: - -* `$WORKING_DIR/.ods/git-dir-commit-sha` which contains the git commit sha of the working directory (no whitespace or newlines). Build tasks can use this to avoid rebuilding when there were no changes in their working dir. +Cleanup will spare directories below the $(cache-deps-parent-dir) so that build tasks must keep their cached files in a subdirectory they create. For example in `$(cache-deps-parent-dir)/go-mods/` where `go-mods` is called technology-name in this proposal. Build tasks supporting caching will also be adjusted to -- Log how long build commands which may be long running take in seconds. +- Log available disk space at the beginning of the build. -- Log available disk space well. This should be implemented so that one can see what sized to require. - -=== Global cache - -Only build tasks for which a global cache provides a significant performance gain will support global caching. At the moment the primary candidates is the go language. Nexus does not support go and we have no similar go dependency artifacts manager in place. - -The following new parameter is introduced to build tasks supporting global caching: - -* `cache-global-require-space-mb` a number defaulting to `0`. This is the number of MB required for the global cache for this build task. If greater 0 caching will be enabled space permitting. - -In addition to these build task parameters, the build tasks also receives the following: - -* File `.ods/cache-global-parent-dir` contains an absolute path without trailing '/' to an existing directory (no whitespace or newlines). If this file does not exits the task must not use global caching as space might have run out and a prior cache may have been deleted. +- Log how long build commands which may be long running take in seconds. -Cleanup will spare directories below the cache-global-parent-dir so that build tasks must keep their cached files in a subdirectory they create. For example in `$(cache-global-parent-dir)/go-modules/` where `go-modules` is called technology-name. +- Log available disk space at the end of the build and also the space delta. -The global cache and workspace directory will be on the same PVC to enable build technologies utilizing the cache without needing to copy files by hard linking. === ods-start and cleanup @@ -89,76 +64,41 @@ Build tasks supporting caching can count on: On the other hand build tasks must not assume that the cache is still available from a prior build. -The cleanup strategy described here is subject to change even in patch versions and specific details must not be relied on. - -Pipeline cleanup happens during `ods-start` +The dependency cache is somewhat special is it typically does not make sense to remove individual files. -The following locations on the PVC are used: +Instead if disk space issues arise one should manually: -- `/ws/` for uncached workspaces. These are recognized if none of there build tasks enabled caching. -- `/.cache-ws//` for cached workspaces. -- `/.cache//` for global caches of a particular build technology. The technology-name would be defined by the build script. +* Increase the PVC space of the associated repository or +* Recreate the PVC (TODO: should this be done automatically for example on a randomized bi-weekly schedule for example) -Before cache cleanup `ods-start` cleans up `/ws/` on the PVC. + -Next all files directly underneath `/.cache/` which are not directories are deleted. This prevents tasks to forget to define and use a technology-name. +However in general cleanup of the PVC does happen during `ods-start`. -**workspace cache cleanup** - -The workspace cache cleanup is skipped if the pipeline has workspace caching disabled to avoid merging workspace cached pipelines into a mainline as describe earlier. - -.TODO this could be done by creating the pipeline with `cache-workspace-require-space-mb` forcefully set to 0 or via another flag. - -`ods-start` then determines: - -- `workspace-required-space-mb-total` :== sum of all declared `cache-workspace-require-space-mb` of build tasks contained in `ods.yaml` - -If this is 0 the next step is to look at global cache cleanup some steps below. - -Otherwise `ods-start` will determine the size of the workspace directory with a variation of `du /.cache-ws//` and convert the result to megabytes. - -Until there is not enough free space: - -- Deleted the oldest workspace. Age should be available via `stat -f '%m' .git/FETCH_HEAD` (see https://stackoverflow.com/a/9229377). -- If there is none continue below. - -If there is still not enough free space: - -- delete `/.cache-ws/` if it exists -- continue the build in `/ws`, so that its `cwd` is at `/ws` and that files `$WORKING_DIR/.ods/git-dir-commit-sha` will not be created for each of the build tasks. - -**global cache cleanup** - -`ods-start` determines: - -- `cache-required-space-mb-total` :== sum of all declared `cache-global-require-space-mb` of build tasks contained in `ods.yaml` +The cleanup strategy described here is subject to change even in patch versions and specific details must not be relied on. -The global cache cleanup deletes in a similar way as the workspace cache cleaning just described but the cleanup candidates are folders at: `/.cache//`. +The following cache locations on the PVC are used: -If there is not sufficient space in the global cache after cleanup the file `.ods/cache-global-parent-dir` will not exist in the workspace directory. +- `/.c/deps//` for dependency caches of a particular build technology. The technology-name would be defined by the build script and unknown to `ods-start`. For dependencies instead of strategy on the file level does not make sense. Instead one should either periodically +- `/.c/*/**` reserved for future usages for example to enable build skipping. For each new supported location `ods-start` may delete files as it makes sense for the purpose. +Before cache cleanup `ods-start` cleans up all files not in the cache locations above. + +Next all files directly underneath `/.c/deps/` which are not directories are deleted. This prevents tasks to forget to define and use a ``. == Pro -* Caching enabled for tools using global cache - -* Workspace caching enables speedup for technologies which benefit most from an incremental build as opposed to a global dependency cache. +* Enables dependency caching. -* Multi build repos can be better supported by avoiding to rebuild when their working directory does not change. +* A global file cache also paves the way for other usages such as caching builds efficiently to enable build skipping. -* Ensuring a mainline build will always be non workspace cached before merging ensures that (workspace) cache issues cannot impact the main build. +* Opt-in to dependency caching makes this a conscious choice from ods-pipeline users. -* Opt-in to caching can be used to document best practices for PVC sizing. +* By only supporting dependency caching in build tools where there is a big benefit we reduce complexity in other cases. -* By having cleanup done in ods-start no need to have an interleaved cleanup process. - -* Having workspaces and caches on the same PVC enables using modern newer technologies such as https://github.com/pnpm/pnpm[pnpm] or https://github.com/yarnpkg/berry[yarn berry (?)] which support deduplication allowing to reference artifacts from a central location without packaging them again. pnpn uses hard links to make this efficient. +* Not implementing file based cleanup for the dependency cache keeps complexity low and increases performance as deleting a to of files can take a lot of time compared to recreating a PVC instead. == Con -* Avoiding unnecessary rebuild of sub builds when their working dir did not change by using workspace caching increases the complexity of the build scripts. Perhaps there is a better way to achieve this as the reports are already available and the build artifacts could be stored in Nexus or even extracted from a prior image. - -* Workspace caching does not increase performance of FE builds much as there the bundling takes typically the the most time. Perhaps newer tools such as https://github.com/evanw/esbuild[esbuild] or https://github.com/yarnpkg/berry[yarn berry] would help to speed up build times without requiring workspace caching. - -* The implementation to ensure mainline builds will always have a non workspace cached build may have bugs and thus the build may still be afflicted by cache issues which might pop up much later. +* By not making dependency cleanup up a responsibility of `ods-start`` users will have to take manual actions if disk space runs out. Here an ability to plug-in an interleaved cleanup process could make sense. + +* Using a file `.ods/cache-deps-parent-dir` enables skipping of using the dependency cache for example via a special tag in a git commit message, but seems otherwise odd. If we find a way to bind the parameters to the task run more explicitly that would be better for traceability. Is there a better way to pass in dynamic parameters from ods-start or the event listener to the build tasks? -* Workspace caching adds complexity (unless we find it can be reduced to an acceptable level) +* Languages where which don't have a mature way to share dependencies globally do not benefit from this approach. For these a workspace caching capability would be needed. From ca5fa5a1ee6946bf4c217a9e94db48684d94bfdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Mon, 14 Feb 2022 16:35:35 +0100 Subject: [PATCH 08/11] correct language snafu and some formatting issue --- docs/proposals/caching.adoc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/proposals/caching.adoc b/docs/proposals/caching.adoc index ac881501..57d72ce6 100644 --- a/docs/proposals/caching.adoc +++ b/docs/proposals/caching.adoc @@ -77,8 +77,9 @@ The cleanup strategy described here is subject to change even in patch versions The following cache locations on the PVC are used: -- `/.c/deps//` for dependency caches of a particular build technology. The technology-name would be defined by the build script and unknown to `ods-start`. For dependencies instead of strategy on the file level does not make sense. Instead one should either periodically -- `/.c/*/**` reserved for future usages for example to enable build skipping. For each new supported location `ods-start` may delete files as it makes sense for the purpose. +- `+/.c/deps//+` for dependency caches of a particular build technology. The technology-name would be defined by the build script and unknown to `ods-start`. + +- `+/.c//**+` reserved for future usages for example to enable build skipping. For each new supported location `ods-start` may delete files as it makes sense for the purpose. Before cache cleanup `ods-start` cleans up all files not in the cache locations above. + Next all files directly underneath `/.c/deps/` which are not directories are deleted. This prevents tasks to forget to define and use a ``. From 6f646a26a4e350670cb4a90dbb12bc9c94d2e425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Wed, 16 Feb 2022 23:03:02 +0100 Subject: [PATCH 09/11] adjust tests --- test/tasks/ods-build-typescript_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/tasks/ods-build-typescript_test.go b/test/tasks/ods-build-typescript_test.go index b8d08ec8..01fd455c 100644 --- a/test/tasks/ods-build-typescript_test.go +++ b/test/tasks/ods-build-typescript_test.go @@ -39,6 +39,8 @@ func TestTaskODSBuildTypescript(t *testing.T) { filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), "docker/dist/src/index.js", + "docker/package.json", + "docker/package-lock.json", ) wantLogMsg := "No sonar-project.properties present, using default:" @@ -132,7 +134,9 @@ func TestTaskODSBuildTypescript(t *testing.T) { filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), - "docker/dist/node_modules/", + "docker/node_modules/", + "docker/package.json", + "docker/package-lock.json", ) }, }, @@ -158,6 +162,8 @@ func TestTaskODSBuildTypescript(t *testing.T) { filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), "docker/dist/src/index.js", + "docker/package.json", + "docker/package-lock.json", ) }, }, From 0a2f5b8a143421c6cf125938fb802a5e65ff1615 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Tue, 22 Feb 2022 14:16:49 +0100 Subject: [PATCH 10/11] small corrections to reflect implementation --- docs/proposals/caching.adoc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/proposals/caching.adoc b/docs/proposals/caching.adoc index 57d72ce6..38e4ddc2 100644 --- a/docs/proposals/caching.adoc +++ b/docs/proposals/caching.adoc @@ -38,7 +38,7 @@ The global cache enables build tasks to store cached files for at least the foll The following new parameter is introduced to build tasks supporting dependency caching: -* `cache-deps` a boolean defaulting to `false`. Only if set to `true` the task will used dependency caching. +* `cache-dependencies` a boolean defaulting to `false`. Only if set to `true` the task will used dependency caching. In addition to the build task parameter, build tasks also receives the following: @@ -77,12 +77,12 @@ The cleanup strategy described here is subject to change even in patch versions The following cache locations on the PVC are used: -- `+/.c/deps//+` for dependency caches of a particular build technology. The technology-name would be defined by the build script and unknown to `ods-start`. +- `+./.ods-cache/deps//+` for dependency caches of a particular build technology. The technology-name would be defined by the build script and unknown to `ods-start`. -- `+/.c//**+` reserved for future usages for example to enable build skipping. For each new supported location `ods-start` may delete files as it makes sense for the purpose. +- `+./.ods-cache//**+` reserved for future usages for example to enable build skipping. For each new supported location `ods-start` may delete files as it makes sense for the purpose. Before cache cleanup `ods-start` cleans up all files not in the cache locations above. + -Next all files directly underneath `/.c/deps/` which are not directories are deleted. This prevents tasks to forget to define and use a ``. +Next all files directly underneath `+./.ods-cache/deps/+` which are not directories are deleted. This prevents tasks to forget to define and use a ``. == Pro From 0b3996d1dedb4e5d37746b9f730e4cd18658e0c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrich=20Kr=C3=A4mer?= Date: Tue, 22 Feb 2022 14:33:51 +0100 Subject: [PATCH 11/11] undo changes no longer needed --- build/package/scripts/build-typescript.sh | 60 +++-------------------- test/tasks/ods-build-typescript_test.go | 8 +-- 2 files changed, 8 insertions(+), 60 deletions(-) diff --git a/build/package/scripts/build-typescript.sh b/build/package/scripts/build-typescript.sh index e462aca6..87f991b1 100755 --- a/build/package/scripts/build-typescript.sh +++ b/build/package/scripts/build-typescript.sh @@ -22,7 +22,6 @@ copyLintReport() { BUILD_DIR="dist" OUTPUT_DIR="docker" WORKING_DIR="." -PIPELINE_CACHE_DIR="" ARTIFACT_PREFIX="" DEBUG="${DEBUG:-false}" MAX_LINT_WARNINGS="0" @@ -35,9 +34,6 @@ while [[ "$#" -gt 0 ]]; do --working-dir) WORKING_DIR="$2"; shift;; --working-dir=*) WORKING_DIR="${1#*=}";; - --pipeline-cache-dir) PIPELINE_CACHE_DIR="$2"; shift;; - --pipeline-cache-dir=*) PIPELINE_CACHE_DIR="${1#*=}";; - --output-dir) OUTPUT_DIR="$2"; shift;; --output-dir=*) OUTPUT_DIR="${1#*=}";; @@ -64,23 +60,9 @@ if [ "${DEBUG}" == "true" ]; then fi ROOT_DIR=$(pwd) -# NOTE: git is currently not available in build script -git_ref=$(cat .ods/git-ref) -case $git_ref in - master|develop) - echo "INFO: not using cache in master or develop branch" - PIPELINE_CACHE_DIR="" - ;; - *);; -esac - - -if [ -d "$PIPELINE_CACHE_DIR" ]; then - # NOTE: rsync is currently not available in build script - echo "Updating cache dir with sources ..." - # rsync -auvhp --itemize-changes --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" - # rsync -auvhp --progress --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" - rsync -auhp --info=progress2 --delete --exclude=node_modules "${WORKING_DIR}" "$PIPELINE_CACHE_DIR" # the info progress2 does not work with verbose, needs rather new rsync version. +if [ "${WORKING_DIR}" != "." ]; then + cd "${WORKING_DIR}" + ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" fi echo "Configuring npm to use Nexus ..." @@ -96,30 +78,14 @@ if [ -n "${NEXUS_HOST}" ] && [ -n "${NEXUS_USERNAME}" ] && [ -n "${NEXUS_PASSWOR npm config set strict-ssl=false fi; -build_dir="${WORKING_DIR}" -output_dir_abs="${ROOT_DIR}/${WORKING_DIR}/${OUTPUT_DIR}" -if [ -d "$PIPELINE_CACHE_DIR" ]; then - build_dir="$PIPELINE_CACHE_DIR/$WORKING_DIR" -fi - -if [ "${WORKING_DIR}" != "." ]; then - ARTIFACT_PREFIX="${WORKING_DIR/\//-}-" -fi -if [ "${build_dir}" != "." ]; then - cd "${build_dir}" -fi - echo "Installing dependencies ..." npm ci --ignore-scripts echo "Linting ..." -start_time=$SECONDS set +e npx eslint src --ext "${LINT_FILE_EXT}" --format compact --max-warnings "${MAX_LINT_WARNINGS}" > eslint-report.txt exitcode=$? set -e -elapsed=$(( SECONDS - start_time )) -echo "linting took $elapsed seconds" if [ $exitcode == 0 ]; then echo "OK" > eslint-report.txt @@ -130,25 +96,16 @@ else fi echo "Building ..." -start_time=$SECONDS npm run build -elapsed=$(( SECONDS - start_time )) -echo "build took $elapsed seconds" -mkdir -p "${output_dir_abs}" -cp -rv "${BUILD_DIR}" "${output_dir_abs}/dist" +mkdir -p "${OUTPUT_DIR}" +cp -r "${BUILD_DIR}" "${OUTPUT_DIR}/dist" if [ "${COPY_NODE_MODULES}" = true ]; then - echo "Copying node_modules to ${output_dir_abs}/dist/node_modules ..." - start_time=$SECONDS - cp -r node_modules "${output_dir_abs}/dist/node_modules" - elapsed=$(( SECONDS - start_time )) - echo "copying node_modules took $elapsed seconds" + echo "Copying node_modules to ${OUTPUT_DIR}/dist/node_modules ..." + cp -r node_modules "${OUTPUT_DIR}/dist/node_modules" fi echo "Testing ..." -# Implement skipping tests for TypeScript #238 -# TODO: is this still needed? -# https://github.com/opendevstack/ods-pipeline/issues/238 if [ -f "${ROOT_DIR}/.ods/artifacts/xunit-reports/${ARTIFACT_PREFIX}report.xml" ]; then echo "Test artifacts already present, skipping tests ..." # Copy artifacts to working directory so that the SonarQube scanner can pick them up later. @@ -174,7 +131,4 @@ else cp build/coverage/lcov.info "${ROOT_DIR}/.ods/artifacts/code-coverage/${ARTIFACT_PREFIX}lcov.info" fi -# Provide disk usage so that one can use this to estimate needs - - supply-sonar-project-properties-default diff --git a/test/tasks/ods-build-typescript_test.go b/test/tasks/ods-build-typescript_test.go index 01fd455c..b8d08ec8 100644 --- a/test/tasks/ods-build-typescript_test.go +++ b/test/tasks/ods-build-typescript_test.go @@ -39,8 +39,6 @@ func TestTaskODSBuildTypescript(t *testing.T) { filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), "docker/dist/src/index.js", - "docker/package.json", - "docker/package-lock.json", ) wantLogMsg := "No sonar-project.properties present, using default:" @@ -134,9 +132,7 @@ func TestTaskODSBuildTypescript(t *testing.T) { filepath.Join(pipelinectxt.SonarAnalysisPath, "analysis-report.md"), filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), - "docker/node_modules/", - "docker/package.json", - "docker/package-lock.json", + "docker/dist/node_modules/", ) }, }, @@ -162,8 +158,6 @@ func TestTaskODSBuildTypescript(t *testing.T) { filepath.Join(pipelinectxt.SonarAnalysisPath, "issues-report.csv"), filepath.Join(pipelinectxt.LintReportsPath, "report.txt"), "docker/dist/src/index.js", - "docker/package.json", - "docker/package-lock.json", ) }, },