diff --git a/.github/actions/wait-for-smoke/action.yml b/.github/actions/wait-for-smoke/action.yml new file mode 100644 index 00000000..b16f9de3 --- /dev/null +++ b/.github/actions/wait-for-smoke/action.yml @@ -0,0 +1,95 @@ +name: 'Wait for Smoke Test' +description: 'Polls the Smoke Test workflow for the current commit and fails if it failed.' + +# Designed to be the leading job in pull_request-triggered workflows so that +# expensive integration CI does not run unless the smoke build passes. +# +# Push events bypass the wait entirely (we still get smoke results for those +# pushes, but other CI is not gated on push). For drafts, callers should +# skip dependent jobs via `if: github.event.pull_request.draft == false` - +# this action will still pass through if smoke is skipped or absent. + +inputs: + workflow: + description: 'Name of the smoke workflow file to wait on' + required: false + default: 'smoke-test.yml' + timeout-seconds: + description: 'Maximum time to wait for smoke to complete' + required: false + default: '1800' + poll-seconds: + description: 'Polling interval' + required: false + default: '20' + github-token: + description: 'GITHUB_TOKEN with actions:read permission' + required: true + +runs: + using: 'composite' + steps: + - name: Wait for smoke + shell: bash + env: + GH_TOKEN: ${{ inputs.github-token }} + SMOKE_WORKFLOW: ${{ inputs.workflow }} + TIMEOUT: ${{ inputs.timeout-seconds }} + POLL: ${{ inputs.poll-seconds }} + REPO: ${{ github.repository }} + run: | + set -u + # Only gate pull_request events. Push events are not gated. + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "Not a pull_request event - skipping smoke gate." + exit 0 + fi + + HEAD_SHA="${{ github.event.pull_request.head.sha }}" + echo "Waiting for $SMOKE_WORKFLOW on $HEAD_SHA (timeout ${TIMEOUT}s)" + + START=$(date +%s) + while :; do + NOW=$(date +%s) + ELAPSED=$((NOW - START)) + if [ "$ELAPSED" -ge "$TIMEOUT" ]; then + echo "::error::Timed out after ${TIMEOUT}s waiting for $SMOKE_WORKFLOW on $HEAD_SHA" + exit 1 + fi + + # Look up the latest run for this workflow + head SHA. + RUN_JSON=$(gh api \ + "repos/${REPO}/actions/workflows/${SMOKE_WORKFLOW}/runs?head_sha=${HEAD_SHA}&per_page=1" \ + 2>/dev/null || echo '{}') + + STATUS=$(echo "$RUN_JSON" | jq -r '.workflow_runs[0].status // "missing"') + CONCLUSION=$(echo "$RUN_JSON" | jq -r '.workflow_runs[0].conclusion // ""') + RUN_URL=$(echo "$RUN_JSON" | jq -r '.workflow_runs[0].html_url // ""') + + case "$STATUS" in + completed) + case "$CONCLUSION" in + success) + echo "Smoke test passed: $RUN_URL" + exit 0 + ;; + skipped|neutral) + echo "Smoke test was $CONCLUSION - treating as pass: $RUN_URL" + exit 0 + ;; + *) + echo "::error::Smoke test concluded as '$CONCLUSION': $RUN_URL" + exit 1 + ;; + esac + ;; + missing) + echo "[$ELAPSED s] No smoke run yet for $HEAD_SHA" + ;; + *) + echo "[$ELAPSED s] Smoke status=$STATUS ($RUN_URL)" + ;; + esac + + sleep "$POLL" + done diff --git a/.github/workflows/_discover-versions.yml b/.github/workflows/_discover-versions.yml new file mode 100644 index 00000000..0dcada65 --- /dev/null +++ b/.github/workflows/_discover-versions.yml @@ -0,0 +1,118 @@ +name: Discover wolfSSL + OpenSSL versions + +# Reusable workflow that resolves at run time: +# - latest wolfSSL v*-stable tag (from upstream wolfssl/wolfssl) +# - Debian Bookworm's stock OpenSSL version (matches what the +# wolfprov-patched .deb on ghcr.io was built against) +# +# Consumers use these outputs to populate matrix values so the +# matrix labels honestly reflect what the test actually installed. +# Today: latest -> v5.8.4-stable, openssl -> 3.0.20 (Bookworm stock). +# When Bookworm bumps OpenSSL or wolfSSL ships a new -stable, the +# resolver picks it up without a CI edit. + +on: + workflow_call: + outputs: + wolfssl_ref: + description: 'Plain string, latest -stable e.g. v5.8.4-stable' + value: ${{ jobs.discover.outputs.wolfssl_ref }} + wolfssl_ref_array: + description: 'JSON array of master + latest -stable for matrix use' + value: ${{ jobs.discover.outputs.wolfssl_ref_array }} + openssl_ref: + description: 'Plain string. Bookworm stock OpenSSL (matches the wolfprov .deb).' + value: ${{ jobs.discover.outputs.openssl_ref }} + openssl_ref_array: + description: 'JSON array form of openssl_ref' + value: ${{ jobs.discover.outputs.openssl_ref_array }} + openssl_latest_ref: + description: 'Plain string, latest upstream openssl-3.x.y release tag (e.g. openssl-3.5.4)' + value: ${{ jobs.discover.outputs.openssl_latest_ref }} + openssl_latest_ref_array: + description: 'JSON array form of openssl_latest_ref' + value: ${{ jobs.discover.outputs.openssl_latest_ref_array }} + openssl_all_releases_array: + description: 'JSON array of every upstream openssl-3.X.Y release tag, sorted ascending. Used by openssl-version.yml so the sweep tracks upstream automatically.' + value: ${{ jobs.discover.outputs.openssl_all_releases_array }} + +jobs: + discover: + name: Resolve wolfSSL + OpenSSL refs + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + wolfssl_ref: ${{ steps.resolve.outputs.wolfssl_ref }} + wolfssl_ref_array: ${{ steps.resolve.outputs.wolfssl_ref_array }} + openssl_ref: ${{ steps.resolve.outputs.openssl_ref }} + openssl_ref_array: ${{ steps.resolve.outputs.openssl_ref_array }} + openssl_latest_ref: ${{ steps.resolve.outputs.openssl_latest_ref }} + openssl_latest_ref_array: ${{ steps.resolve.outputs.openssl_latest_ref_array }} + openssl_all_releases_array: ${{ steps.resolve.outputs.openssl_all_releases_array }} + steps: + - name: Resolve versions + id: resolve + run: | + set -euo pipefail + + # ---- wolfSSL: highest v*-stable tag from upstream ---- + WOLFSSL=$(git ls-remote --tags --refs https://github.com/wolfSSL/wolfssl.git 'v*-stable' \ + | awk -F/ '{print $NF}' | sort -V | tail -n 1) + if [ -z "${WOLFSSL:-}" ]; then + echo "::error::Could not resolve latest wolfSSL -stable tag" + exit 1 + fi + + # ---- OpenSSL (Debian Bookworm stock) ---- + # The wolfprov-patched .deb on ghcr.io is built by patching + # Bookworm's stock libssl3 source, so this is the actual + # OpenSSL the Debian-container workflows end up linking against. + # Use docker to ask Bookworm's apt directly, then strip the + # Debian revision (3.0.20-1~deb12u1 -> 3.0.20). + OSSL_RAW=$(docker run --rm debian:bookworm sh -c \ + 'apt-get update -qq >/dev/null 2>&1 && apt-cache madison openssl | head -1' \ + | awk '{print $3}') + if [ -z "${OSSL_RAW:-}" ]; then + echo "::error::Could not resolve Bookworm OpenSSL version" + exit 1 + fi + OSSL=$(echo "$OSSL_RAW" | sed 's/-.*//') + + # ---- OpenSSL (all upstream release tags, sorted) ---- + # Used by openssl-version.yml so the sweep tracks upstream + # automatically as new releases ship. Release-shaped only: + # openssl-X.Y.Z, no -alpha/-beta/-pre. Floored at the + # historical oldest-supported version below so we don't + # silently re-introduce coverage of openssl-3.0.0/3.0.1/3.0.2 + # that the static matrix used to exclude. + OSSL_FLOOR="openssl-3.0.3" + OSSL_ALL=$(git ls-remote --tags --refs https://github.com/openssl/openssl.git 'openssl-3.*' \ + | awk -F/ '{print $NF}' \ + | grep -E '^openssl-3\.[0-9]+\.[0-9]+$' \ + | sort -V \ + | awk -v floor="$OSSL_FLOOR" '$0 == floor {p=1} p') + if [ -z "${OSSL_ALL:-}" ]; then + echo "::error::Could not resolve upstream OpenSSL release tags (floor=$OSSL_FLOOR)" + exit 1 + fi + # JSON array. jq -R reads each line as a string, -s collects + # them into an array, -c emits compact single-line JSON. + OSSL_ALL_JSON=$(printf '%s\n' "$OSSL_ALL" | jq -R . | jq -s -c .) + # Highest version (last after sort -V) is the resolved + # "latest" used by source-built workflows. + OSSL_LATEST=$(echo "$OSSL_ALL" | tail -n 1) + + echo "wolfSSL latest -stable: $WOLFSSL (also testing master)" + echo "Bookworm OpenSSL: openssl-$OSSL (raw: $OSSL_RAW)" + echo "Upstream OpenSSL latest: $OSSL_LATEST" + echo "Upstream OpenSSL releases ($(echo "$OSSL_ALL" | wc -l) tags)" + + { + echo "wolfssl_ref=$WOLFSSL" + echo "wolfssl_ref_array=[\"master\",\"$WOLFSSL\"]" + echo "openssl_ref=openssl-$OSSL" + echo "openssl_ref_array=[\"openssl-$OSSL\"]" + echo "openssl_latest_ref=$OSSL_LATEST" + echo "openssl_latest_ref_array=[\"$OSSL_LATEST\"]" + echo "openssl_all_releases_array=$OSSL_ALL_JSON" + } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/bind9.yml b/.github/workflows/bind9.yml index 264bf1f7..fa141948 100644 --- a/.github/workflows/bind9.yml +++ b/.github/workflows/bind9.yml @@ -1,19 +1,20 @@ name: Bind9 Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Bind9 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,17 +22,18 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_bind: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. @@ -40,10 +42,9 @@ jobs: fail-fast: false matrix: bind_ref: [ 'v9.18.28' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -85,16 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install bind9 test dependencies - run: | - apt-get update - apt install -y build-essential automake libtool gnutls-bin \ - pkg-config make libidn2-dev libuv1-dev libnghttp2-dev libcap-dev \ - libjemalloc-dev zlib1g-dev libxml2-dev libjson-c-dev libcmocka-dev \ - python3-pytest python3-dnspython python3-hypothesis patch iproute2 \ - net-tools git - PERL_MM_USE_DEFAULT=1 cpan -i Net::DNS - - name: Checkout bind9 uses: actions/checkout@v4 with: @@ -129,7 +120,13 @@ jobs: make -j$(nproc) ./bin/tests/system/ifconfig.sh up - export ${{ matrix.force_fail }} + # --- normal mode --- + make -j$(nproc) check 2>&1 | tee bind9-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" bind9 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 make -j$(nproc) check 2>&1 | tee bind9-test.log TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} bind9 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" bind9 diff --git a/.github/workflows/build-wolfprovider.yml b/.github/workflows/build-wolfprovider.yml index 9c9bb9c5..dcef8e06 100644 --- a/.github/workflows/build-wolfprovider.yml +++ b/.github/workflows/build-wolfprovider.yml @@ -90,7 +90,15 @@ jobs: run: | ORAS_VERSION="1.2.2" ORAS_CHECKSUM="bff970346470e5ef888e9f2c0bf7f8ee47283f5a45207d6e7a037da1fb0eae0d" - curl -sLO "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz" + # GitHub releases are an occasional flake point; retry the download. + rm -f "oras_${ORAS_VERSION}_linux_amd64.tar.gz" + for attempt in 1 2 3; do + if curl -fsSLO "https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}/oras_${ORAS_VERSION}_linux_amd64.tar.gz"; then + break + fi + echo "ORAS download attempt $attempt failed." + [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) + done echo "${ORAS_CHECKSUM} oras_${ORAS_VERSION}_linux_amd64.tar.gz" | sha256sum -c - || { echo "ERROR: ORAS checksum verification failed!" exit 1 @@ -109,32 +117,53 @@ jobs: --password-stdin ghcr.io # ── Debian build: pull .deb packages from ghcr.io ── + # The wolfprov debs (ghcr.io/wolfssl/wolfprovider/debs:*) are + # private. Only canonical-repo runs have a token authorized to + # pull them; forks would 401 and silently skip the install, so + # short-circuit here to fail loud and explain. - name: Download pre-built packages from ghcr.io if: steps.check_artifact.outcome != 'success' && github.repository == 'wolfSSL/wolfProvider' && inputs.build_type == 'debian' run: | mkdir -p ${{ env.WOLFSSL_PACKAGES_PATH }} mkdir -p ${{ env.OPENSSL_PACKAGES_PATH }} + # Retry ORAS pulls -- ghcr.io is the single biggest flake source + # in this CI, so don't let one transient network blip kill the + # whole matrix. + oras_pull_with_retry() { + local image="$1" + local outdir="$2" + for attempt in 1 2 3; do + if oras pull "$image" -o "$outdir"; then + return 0 + fi + echo "oras pull $image attempt $attempt failed." + [ "$attempt" -lt 3 ] && sleep $((attempt * 15)) + done + echo "ERROR: oras pull $image failed after 3 attempts." + return 1 + } + # Pull wolfSSL packages based on FIPS variant if [ "${{ inputs.fips_ref }}" = "FIPS" ]; then echo "Pulling FIPS wolfSSL packages..." - oras pull ghcr.io/wolfssl/wolfprovider/debs:fips \ - -o ${{ env.WOLFSSL_PACKAGES_PATH }} + oras_pull_with_retry ghcr.io/wolfssl/wolfprovider/debs:fips \ + ${{ env.WOLFSSL_PACKAGES_PATH }} else echo "Pulling non-FIPS wolfSSL packages..." - oras pull ghcr.io/wolfssl/wolfprovider/debs:nonfips \ - -o ${{ env.WOLFSSL_PACKAGES_PATH }} + oras_pull_with_retry ghcr.io/wolfssl/wolfprovider/debs:nonfips \ + ${{ env.WOLFSSL_PACKAGES_PATH }} fi # Pull OpenSSL packages based on replace_default setting if [ "${{ inputs.replace_default }}" = "true" ]; then echo "Pulling OpenSSL replace-default packages..." - oras pull ghcr.io/wolfssl/wolfprovider/debs:openssl-replace-default \ - -o ${{ env.OPENSSL_PACKAGES_PATH }} + oras_pull_with_retry ghcr.io/wolfssl/wolfprovider/debs:openssl-replace-default \ + ${{ env.OPENSSL_PACKAGES_PATH }} else echo "Pulling OpenSSL default packages..." - oras pull ghcr.io/wolfssl/wolfprovider/debs:openssl-default \ - -o ${{ env.OPENSSL_PACKAGES_PATH }} + oras_pull_with_retry ghcr.io/wolfssl/wolfprovider/debs:openssl-default \ + ${{ env.OPENSSL_PACKAGES_PATH }} fi # Validate that we actually got .deb files @@ -164,8 +193,13 @@ jobs: - name: Install xz-utils if: steps.check_artifact.outcome != 'success' && inputs.build_type == 'yocto' run: | - apt-get update - apt-get install -y xz-utils + for attempt in 1 2 3; do + if apt-get update && apt-get install -y xz-utils; then + break + fi + echo "apt attempt $attempt failed." + [ "$attempt" -lt 3 ] && sleep $((attempt * 10)) + done - name: Download WIC images from ghcr.io if: steps.check_artifact.outcome != 'success' && github.repository == 'wolfSSL/wolfProvider' && inputs.build_type == 'yocto' @@ -174,8 +208,14 @@ jobs: TAG="${{ steps.prepare_artifact_name.outputs.fips_str }}-${{ steps.prepare_artifact_name.outputs.config_str }}" echo "Pulling ghcr.io/wolfssl/wolfprovider/wics:${TAG}..." - oras pull "ghcr.io/wolfssl/wolfprovider/wics:${TAG}" \ - -o ${{ env.YOCTO_IMAGES_PATH }} + for attempt in 1 2 3; do + if oras pull "ghcr.io/wolfssl/wolfprovider/wics:${TAG}" \ + -o ${{ env.YOCTO_IMAGES_PATH }}; then + break + fi + echo "WIC pull attempt $attempt failed." + [ "$attempt" -lt 3 ] && sleep $((attempt * 15)) + done cd ${{ env.YOCTO_IMAGES_PATH }} diff --git a/.github/workflows/cjose.yml b/.github/workflows/cjose.yml index 3d593a89..61e1977a 100644 --- a/.github/workflows/cjose.yml +++ b/.github/workflows/cjose.yml @@ -1,19 +1,20 @@ name: cjose Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for cjose Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,42 +22,37 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_cjose: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # Run inside Debian Bookworm to match packaging environment container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: # Dont test osp master since it might be unstable cjose_ref: [ 'v0.6.2.1' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages steps: - - name: Install cjose dependencies - run: | - apt-get update - apt-get install -y git build-essential autoconf automake \ - libtool pkg-config libjansson-dev check ca-certificates dpkg-dev - - name: Checkout wolfProvider uses: actions/checkout@v4 with: @@ -111,9 +107,17 @@ jobs: - name: Run cjose tests working-directory: cjose run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + + make test 2>&1 | tee cjose-test.log + TEST_RESULT=$(grep -q "FAIL: check_cjose" cjose-test.log && echo "1" || echo "0") + echo "TEST_RESULT = $TEST_RESULT" + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" cjose + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 make test 2>&1 | tee cjose-test.log TEST_RESULT=$(grep -q "FAIL: check_cjose" cjose-test.log && echo "1" || echo "0") echo "TEST_RESULT = $TEST_RESULT" - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} cjose + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" cjose diff --git a/.github/workflows/cmdline.yml b/.github/workflows/cmdline.yml index 7183fcaf..8e49a38b 100644 --- a/.github/workflows/cmdline.yml +++ b/.github/workflows/cmdline.yml @@ -6,6 +6,18 @@ on: branches: [ 'master', 'main', 'release/**' ] pull_request: branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -13,16 +25,25 @@ concurrency: # END OF COMMON SECTION jobs: + discover_versions: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + uses: ./.github/workflows/_discover-versions.yml + cmdtest_test: + needs: discover_versions + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false name: Command line test runs-on: ubuntu-22.04 timeout-minutes: 20 strategy: + fail-fast: false matrix: - openssl_ref: [ 'master', 'openssl-3.5.0' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] + openssl_ref: + - master + - ${{ needs.discover_versions.outputs.openssl_latest_ref }} + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} debug: ['WOLFPROV_DEBUG=1', ''] + # force_fail collapsed into sequential test runs below steps: - name: Checkout wolfProvider uses: actions/checkout@v4 @@ -36,4 +57,9 @@ jobs: - name: Run tests run: | source scripts/env-setup - ${{ matrix.force_fail }} ${{ matrix.debug }} ./scripts/cmd_test/do-cmd-tests.sh + + # --- normal mode --- + ${{ matrix.debug }} ./scripts/cmd_test/do-cmd-tests.sh + + # --- force-fail mode --- + WOLFPROV_FORCE_FAIL=1 ${{ matrix.debug }} ./scripts/cmd_test/do-cmd-tests.sh diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 374e61bc..7618aac6 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -6,6 +6,18 @@ on: branches: [ 'master', 'main', 'release/**' ] pull_request: branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -14,6 +26,7 @@ concurrency: jobs: codespell: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false name: Check for spelling errors runs-on: ubuntu-22.04 timeout-minutes: 5 diff --git a/.github/workflows/curl.yml b/.github/workflows/curl.yml index 5f49d55f..51749070 100644 --- a/.github/workflows/curl.yml +++ b/.github/workflows/curl.yml @@ -1,19 +1,20 @@ name: Curl Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Curl Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,29 +22,31 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_curl: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: curl_ref: [ 'curl-8_4_0', 'curl-7_88_1' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -84,12 +87,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y nghttp2 libpsl5 libpsl-dev python3-impacket \ - build-essential autoconf automake libtool - - name: Build curl uses: wolfSSL/actions-build-autotools-project@v1 with: @@ -101,25 +98,29 @@ jobs: - name: Generate certificates for curl master force-fail tests run: | - if [ "${{ matrix.force_fail }}" = "WOLFPROV_FORCE_FAIL=1" ] && - [ "${{ matrix.curl_ref }}" = "master" ]; then + # Only curl master needs these test certs for force-fail mode. + if [ "${{ matrix.curl_ref }}" = "master" ]; then cd curl/tests/certs make test-ca.cacert cd ../.. fi + - name: Test curl with wolfProvider working-directory: curl shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} export CURL_REF=${{ matrix.curl_ref }} - # Tests rely on $USER being set export USER=testuser - # Run tests and save output to test.log + # --- normal mode --- + make -j$(nproc) test-ci 2>&1 | tee curl-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" curl + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 make -j$(nproc) test-ci 2>&1 | tee curl-test.log - # Capture the test result using PIPESTATUS (Bash only) TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} curl + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" curl diff --git a/.github/workflows/debian-package.yml b/.github/workflows/debian-package.yml index 2be668d5..f0265b52 100644 --- a/.github/workflows/debian-package.yml +++ b/.github/workflows/debian-package.yml @@ -1,19 +1,20 @@ name: Debian Package Test -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Debian Package Test. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,30 +22,32 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true, false ] libwolfprov-replace-default: name: libwolfprov ${{ matrix.replace_default && 'replace-default' || 'standalone' }} ${{ matrix.fips_ref }} runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # Run inside Debian Bookworm to match packaging environment container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true, false ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -70,12 +73,9 @@ jobs: ${{ env.OPENSSL_PACKAGES_PATH }}/openssl_*.deb \ ${{ env.OPENSSL_PACKAGES_PATH }}/libssl3_*.deb \ ${{ env.OPENSSL_PACKAGES_PATH }}/libssl-dev_*.deb - else - # Install standard OpenSSL packages - apt-get update - apt-get install -y \ - openssl libssl3 libssl-dev fi + # Standalone mode uses the stock openssl/libssl3/libssl-dev that + # ship in the wolfprovider-test-deps container -- no install needed. - name: Install wolfSSL and wolfProvider packages run: | @@ -107,8 +107,15 @@ jobs: shell: bash run: | # Run the do-cmd-test.sh script to execute interoperability tests - echo "Running OpenSSL provider interoperability tests..." - OPENSSL_BIN=$(eval which openssl) ${{ matrix.replace_default && 'WOLFPROV_REPLACE_DEFAULT=1' || '' }} ${{ matrix.force_fail }} ${{ matrix.fips_ref == 'FIPS' && 'WOLFSSL_ISFIPS=1' || '' }} ./scripts/cmd_test/do-cmd-tests.sh + export OPENSSL_BIN=$(eval which openssl) + REPLACE_DEFAULT="${{ matrix.replace_default && 'WOLFPROV_REPLACE_DEFAULT=1' || '' }}" + ISFIPS="${{ matrix.fips_ref == 'FIPS' && 'WOLFSSL_ISFIPS=1' || '' }}" + + echo "Running interoperability tests (normal mode)..." + env $REPLACE_DEFAULT $ISFIPS ./scripts/cmd_test/do-cmd-tests.sh + + echo "Running interoperability tests (force-fail mode)..." + env $REPLACE_DEFAULT WOLFPROV_FORCE_FAIL=1 $ISFIPS ./scripts/cmd_test/do-cmd-tests.sh echo "PASS: All provider interoperability tests successful" - name: Uninstall package and verify cleanup diff --git a/.github/workflows/fips-ready.yml b/.github/workflows/fips-ready.yml index d5d0d1e1..8e8560c1 100644 --- a/.github/workflows/fips-ready.yml +++ b/.github/workflows/fips-ready.yml @@ -6,6 +6,18 @@ on: branches: [ 'master', 'main', 'release/**' ] pull_request: branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -13,15 +25,22 @@ concurrency: # END OF COMMON SECTION jobs: + discover_versions: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + uses: ./.github/workflows/_discover-versions.yml + fips_ready_test: + needs: discover_versions + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false name: FIPS Ready Bundle Test runs-on: ubuntu-22.04 timeout-minutes: 20 strategy: + fail-fast: false matrix: wolfssl_bundle_ref: [ '5.8.2' ] - openssl_ref: [ 'openssl-3.5.0' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_latest_ref_array) }} + # force_fail collapsed into sequential runs in the test step steps: - name: Checkout wolfProvider uses: actions/checkout@v4 @@ -56,7 +75,10 @@ jobs: run: | # Run cmd tests to verify functionality export WOLFSSL_ISFIPS=1 - export ${{matrix.force_fail}} source scripts/env-setup - ${{ matrix.force_fail }} ./scripts/cmd_test/do-cmd-tests.sh + # --- normal mode --- + ./scripts/cmd_test/do-cmd-tests.sh + + # --- force-fail mode --- + WOLFPROV_FORCE_FAIL=1 ./scripts/cmd_test/do-cmd-tests.sh diff --git a/.github/workflows/git-ssh-dr.yml b/.github/workflows/git-ssh-dr.yml index 881f5b33..2305d573 100644 --- a/.github/workflows/git-ssh-dr.yml +++ b/.github/workflows/git-ssh-dr.yml @@ -1,17 +1,20 @@ name: Git SSH Default Replace Tests -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Git SSH Default Replace Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -19,30 +22,32 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] git-ssh-default-replace-test: runs-on: ubuntu-22.04 container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] key_type: [ 'rsa', 'ecdsa', 'ed25519', 'chacha20-poly1305' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] - iterations: [ 10 ] # Total of 50 runs + iterations: [ 10 ] # Total of 80 runs + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -84,13 +89,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Set up environment - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y openssh-client openssh-server expect xxd git \ - net-tools git-all - - name: Run git + replace default + ssh test shell: bash run: | @@ -100,8 +98,14 @@ jobs: echo "Testing with key type: ${{ matrix.key_type }}" echo "Running ${{ matrix.iterations }} iterations" - # Run the scripts test - ${{ matrix.force_fail }} ./scripts/test-git-ssh-dr.sh \ + # --- normal mode --- + ./scripts/test-git-ssh-dr.sh \ + --key-types "${{ matrix.key_type }}" \ + --iterations "${{ matrix.iterations }}" \ + --verbose + + # --- force-fail mode --- + WOLFPROV_FORCE_FAIL=1 ./scripts/test-git-ssh-dr.sh \ --key-types "${{ matrix.key_type }}" \ --iterations "${{ matrix.iterations }}" \ --verbose diff --git a/.github/workflows/grpc.yml b/.github/workflows/grpc.yml index 1761a66c..b8f9357b 100644 --- a/.github/workflows/grpc.yml +++ b/.github/workflows/grpc.yml @@ -1,19 +1,20 @@ name: gRPC Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for gRPC Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,17 +22,18 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_grpc: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. @@ -47,11 +49,11 @@ jobs: ssl_transport_security_test ssl_transport_security_utils_test test_core_security_ssl_credentials_test test_cpp_end2end_ssl_credentials_test h2_ssl_cert_test h2_ssl_session_reuse_test - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -92,12 +94,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install prerequisites - run: | - apt-get update - apt-get install -y build-essential autoconf libtool pkg-config clang \ - libc++-dev iproute2 net-tools git python3-six - - name: Confirm IPv4 and IPv6 support run: | ip addr list lo | grep 'inet ' @@ -149,33 +145,37 @@ jobs: # Start the port server ./tools/run_tests/start_port_server.py - export ${{ matrix.force_fail }} - set +e - - # Run the tests - all_passed=1 - for t in ${{ matrix.tests }} ; do - echo "===================================" - echo "Running test: $t" - echo "Force fail: ${{ matrix.force_fail }}" - echo "===================================" - ./cmake/build/$t - exit_code=$? - if [ $exit_code -ne 0 ]; then - echo "Test $t FAILED with exit code $exit_code" - echo "Force fail: ${{ matrix.force_fail }}" - all_passed=0 + run_grpc_round() { + local mode_label="$1" + local ff_arg="$2" + local all_passed=1 + set +e + for t in ${{ matrix.tests }} ; do + echo "===================================" + echo "Running test: $t (mode: $mode_label)" + echo "===================================" + ./cmake/build/$t + local exit_code=$? + if [ $exit_code -ne 0 ]; then + echo "Test $t FAILED with exit code $exit_code (mode: $mode_label)" + all_passed=0 + fi + done + set -e + local result + if [ $all_passed -eq 1 ]; then + result=0 + echo "ALL TESTS PASSED (mode: $mode_label)" + else + result=1 + echo "SOME TESTS FAILED (mode: $mode_label)" fi - done - - set -e - if [ $all_passed -eq 1 ]; then - echo "ALL TESTS PASSED" - TEST_RESULT=0 - else - echo "SOME TESTS FAILED" - TEST_RESULT=1 - fi - - # Check results - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} grpc + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $result "$ff_arg" grpc + } + + # --- normal mode --- + run_grpc_round normal "" + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + run_grpc_round ff "WOLFPROV_FORCE_FAIL=1" diff --git a/.github/workflows/hostap.yml b/.github/workflows/hostap.yml index 370709f7..18eed4d0 100644 --- a/.github/workflows/hostap.yml +++ b/.github/workflows/hostap.yml @@ -1,19 +1,20 @@ name: hostap and wpa supplicant Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**'] - pull_request: - branches: [ '*' ] +# OSP integration test for hostap and wpa supplicant Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,31 +22,33 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_hostap: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # Run inside Debian Bookworm with privileged access for UML container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm options: --privileged --cap-add=ALL -v /dev:/dev env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 90 strategy: + fail-fast: false matrix: hostap_ref: [ 'main' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] + # force_fail collapsed into sequential VM test rounds below env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -98,15 +101,9 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install hostap dependencies + - name: Install hostap-specific Python crypto module run: | - apt-get update - apt-get install -y libpcap0.8 libpcap-dev curl libcurl4-openssl-dev \ - libnl-3-dev binutils-dev libiberty-dev libnl-genl-3-dev libnl-route-3-dev \ - libdbus-1-dev bridge-utils tshark python3-pycryptodome libsqlite3-dev \ - libzstd1 wireless-tools iw build-essential autoconf automake libtool \ - pkg-config git wget ca-certificates flex bison bc libxml2-dev zlib1g-dev \ - python3-pip psmisc iproute2 procps net-tools systemd kmod wireless-regdb + # hostap test scripts need pip's cryptography, not the apt one. apt-get remove -y python3-cryptography 2>/dev/null || true pip install --no-cache-dir --force-reinstall --break-system-packages cryptography @@ -154,27 +151,10 @@ jobs: LIBS += -lssl -lcrypto EOF - - name: Setup non-WPFF environment - working-directory: hostap/tests/hwsim - if: matrix.force_fail == '' - run: | - cd vm && git checkout inside.sh 2>/dev/null || true && cd .. - sed -i '115 r /dev/stdin' vm/inside.sh <<'ENVEOF' - cat > /tmp/bin/halt << 'HALTEOF' - #!/bin/sh - sync - exit 0 - HALTEOF - chmod +x /tmp/bin/halt - OPENSSL_MODULES_PATH=$(find /usr -name "libwolfprov.so" -exec dirname {} \; 2>/dev/null | head -1) - [ -n "$OPENSSL_MODULES_PATH" ] && export OPENSSL_MODULES="$OPENSSL_MODULES_PATH" - export OPENSSL_CONF="/etc/ssl/openssl.cnf" - export CRYPTOGRAPHY_OPENSSL_NO_LEGACY=1 - ENVEOF - - - name: Setup WPFF environment + # The non-WPFF env writes vm/inside.sh without WOLFPROV_FORCE_FAIL=1. + # The WPFF round below re-writes vm/inside.sh with that export added. + - name: Setup non-WPFF environment (round 1) working-directory: hostap/tests/hwsim - if: matrix.force_fail == 'WOLFPROV_FORCE_FAIL=1' run: | cd vm && git checkout inside.sh 2>/dev/null || true && cd .. sed -i '115 r /dev/stdin' vm/inside.sh <<'ENVEOF' @@ -188,7 +168,6 @@ jobs: [ -n "$OPENSSL_MODULES_PATH" ] && export OPENSSL_MODULES="$OPENSSL_MODULES_PATH" export OPENSSL_CONF="/etc/ssl/openssl.cnf" export CRYPTOGRAPHY_OPENSSL_NO_LEGACY=1 - export WOLFPROV_FORCE_FAIL=1 ENVEOF - name: Update certs @@ -205,10 +184,9 @@ jobs: ldd hostapd/hostapd | grep ssl ldd wpa_supplicant/wpa_supplicant | grep ssl - - name: Run focused tests + - name: Run focused tests (normal mode) id: testing working-directory: hostap/tests/hwsim/ - continue-on-error: true run: | set +e @@ -216,6 +194,7 @@ jobs: # Run smoke tests SMOKE_TESTS="ap_open ap_wpa2_psk discovery" + rm -rf /tmp/hwsim-test-logs timeout 3m ./vm/parallel-vm.py --nocurses $(nproc) $SMOKE_TESTS || SMOKE_RES=$? # Run EAP tests (excluding MSCHAPv2 - requires MD4/DES not in wolfSSL) @@ -228,9 +207,6 @@ jobs: FINAL_RES=1 fi - # Check for connection failures (common with WOLFPROV_FORCE_FAIL) - WPA_CONNECT_FAILS=$(grep -h "Could not connect to /tmp/wpas" /tmp/hwsim-test-logs/*-parallel.log 2>/dev/null | wc -l || echo "0") - # Ignore NOT-FOUND errors (test files missing/require special params) NOT_FOUND=$(grep -h "NOT-FOUND" /tmp/hwsim-test-logs/*-parallel.log 2>/dev/null | wc -l || echo "0") REAL_FAILS=$(grep -h "Failed:" /tmp/hwsim-test-logs/*-parallel.log 2>/dev/null | grep -v "NOT-FOUND" | wc -l || echo "0") @@ -238,22 +214,59 @@ jobs: FINAL_RES=0 fi - # Check results based on test mode - if [ "${{ matrix.force_fail }}" = "WOLFPROV_FORCE_FAIL=1" ]; then - # With force fail, we expect failures or connection issues - if [ $FINAL_RES -ne 0 ] || [ "$WPA_CONNECT_FAILS" -gt "0" ]; then - echo "✓ EXPECTED: Tests failed/crashed with WOLFPROV_FORCE_FAIL=1" - exit 0 - else - echo "✗ UNEXPECTED: Tests passed with WOLFPROV_FORCE_FAIL=1" - exit 1 - fi + if [ $FINAL_RES -eq 0 ]; then + echo "✓ SUCCESS: wolfProvider tests passed" + exit 0 + else + echo "✗ FAILURE: wolfProvider tests failed" + exit 1 + fi + + - name: Setup WPFF environment (round 2) + working-directory: hostap/tests/hwsim + run: | + cd vm && git checkout inside.sh 2>/dev/null || true && cd .. + sed -i '115 r /dev/stdin' vm/inside.sh <<'ENVEOF' + cat > /tmp/bin/halt << 'HALTEOF' + #!/bin/sh + sync + exit 0 + HALTEOF + chmod +x /tmp/bin/halt + OPENSSL_MODULES_PATH=$(find /usr -name "libwolfprov.so" -exec dirname {} \; 2>/dev/null | head -1) + [ -n "$OPENSSL_MODULES_PATH" ] && export OPENSSL_MODULES="$OPENSSL_MODULES_PATH" + export OPENSSL_CONF="/etc/ssl/openssl.cnf" + export CRYPTOGRAPHY_OPENSSL_NO_LEGACY=1 + export WOLFPROV_FORCE_FAIL=1 + ENVEOF + + - name: Run focused tests (force-fail mode) + working-directory: hostap/tests/hwsim/ + run: | + set +e + + SMOKE_RES=0 + TLS_RES=0 + + SMOKE_TESTS="ap_open ap_wpa2_psk discovery" + rm -rf /tmp/hwsim-test-logs + timeout 3m ./vm/parallel-vm.py --nocurses $(nproc) $SMOKE_TESTS || SMOKE_RES=$? + + TLS_EAP_TESTS="ap_wpa2_eap_tls ap_wpa2_eap_ttls_eap_gtc ap_wpa2_eap_peap_eap_tls" + timeout 5m ./vm/parallel-vm.py --nocurses $(nproc) $TLS_EAP_TESTS || TLS_RES=$? + + FINAL_RES=0 + if [ "${SMOKE_RES:-0}" -ne "0" ] || [ "${TLS_RES:-0}" -ne "0" ]; then + FINAL_RES=1 + fi + + WPA_CONNECT_FAILS=$(grep -h "Could not connect to /tmp/wpas" /tmp/hwsim-test-logs/*-parallel.log 2>/dev/null | wc -l || echo "0") + + # With force fail, we expect failures or connection issues + if [ $FINAL_RES -ne 0 ] || [ "$WPA_CONNECT_FAILS" -gt "0" ]; then + echo "✓ EXPECTED: Tests failed/crashed with WOLFPROV_FORCE_FAIL=1" + exit 0 else - if [ $FINAL_RES -eq 0 ]; then - echo "✓ SUCCESS: wolfProvider tests passed" - exit 0 - else - echo "✗ FAILURE: wolfProvider tests failed" - exit 1 - fi + echo "✗ UNEXPECTED: Tests passed with WOLFPROV_FORCE_FAIL=1" + exit 1 fi diff --git a/.github/workflows/iperf.yml b/.github/workflows/iperf.yml index 5f9c3c3d..b900a4ef 100644 --- a/.github/workflows/iperf.yml +++ b/.github/workflows/iperf.yml @@ -1,19 +1,20 @@ name: iperf Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for iperf Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_iperf: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: iperf_ref: [ '3.12' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,12 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y build-essential autoconf libtool pkg-config clang \ - libc++-dev - - name: Checkout iperf uses: actions/checkout@v4 with: @@ -122,24 +118,34 @@ jobs: - name: Run tests working-directory: iperf run: | - export ${{ matrix.force_fail }} - - # Test variables for iperf + # Test variables for iperf (shared across both modes) export IPERF3_EXECUTABLE=$GITHUB_WORKSPACE/iperf/src/iperf3 export IPERF3_LIB=$GITHUB_WORKSPACE/iperf/src/.libs/libiperf.so export IPERF3_TEST_INTERVAL=0.1 export IPERF3_TEST_DURATION=10 - export IPERF3_TEST_LOG=iperf-test.log export IPERF3_USER=mario export IPERF3_PASSWORD=rossi export KEY_DIR=$GITHUB_WORKSPACE/test-keys - # Launch the iperf server in the background - $IPERF3_EXECUTABLE -s \ - --rsa-private-key-path $KEY_DIR/rsa_private_unprotected.pem \ - --authorized-users-path $KEY_DIR/credentials.csv & - - # Run the client - $IPERF3_EXECUTABLE -c localhost -i $IPERF3_TEST_INTERVAL -t $IPERF3_TEST_DURATION \ - --rsa-public-key-path $KEY_DIR/rsa_public.pem \ - --user $IPERF3_USER | tee $IPERF3_TEST_LOG \ + run_iperf_round() { + local mode="$1" + local log="iperf-test-${mode}.log" + # Server in the background per round + $IPERF3_EXECUTABLE -s \ + --rsa-private-key-path $KEY_DIR/rsa_private_unprotected.pem \ + --authorized-users-path $KEY_DIR/credentials.csv & + local server_pid=$! + sleep 1 + $IPERF3_EXECUTABLE -c localhost -i $IPERF3_TEST_INTERVAL -t $IPERF3_TEST_DURATION \ + --rsa-public-key-path $KEY_DIR/rsa_public.pem \ + --user $IPERF3_USER | tee "$log" + kill "$server_pid" 2>/dev/null || true + wait "$server_pid" 2>/dev/null || true + } + + # --- normal mode --- + run_iperf_round normal + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + run_iperf_round ff diff --git a/.github/workflows/krb5.yml b/.github/workflows/krb5.yml index 05f0d7be..c37a7b61 100644 --- a/.github/workflows/krb5.yml +++ b/.github/workflows/krb5.yml @@ -1,19 +1,20 @@ name: KRB5 Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for KRB5 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_krb5: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 30 strategy: + fail-fast: false matrix: krb5_ref: [ 'krb5-1.20.1-final' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,15 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install KRB5 dependencies - run: | - apt-get update - apt-get install -y \ - build-essential autoconf automake libtool \ - bison flex libldap2-dev libkeyutils-dev \ - libverto-dev libcom-err2 comerr-dev \ - libss2 ss-dev - - name: Checkout KRB5 uses: actions/checkout@v4 with: @@ -135,9 +128,17 @@ jobs: make -j$(nproc) make install - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run tests and save output + make check 2>&1 | tee krb5-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" krb5 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests and save output make check 2>&1 | tee krb5-test.log TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} krb5 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" krb5 diff --git a/.github/workflows/libcryptsetup.yml b/.github/workflows/libcryptsetup.yml index 6727bf5e..22504c73 100644 --- a/.github/workflows/libcryptsetup.yml +++ b/.github/workflows/libcryptsetup.yml @@ -1,19 +1,20 @@ name: Libcryptsetup Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Libcryptsetup Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,17 +22,18 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_cryptsetup: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. @@ -40,10 +42,9 @@ jobs: fail-fast: false matrix: cryptsetup_ref: [ 'v2.6.1' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -86,14 +87,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y \ - build-essential autoconf asciidoctor gettext autopoint libtool \ - pkg-config uuid-dev libdevmapper-dev libpopt-dev libjson-c-dev \ - libargon2-dev libblkid-dev bsdextrautils kmod util-linux cryptsetup-bin - - name: Checkout cryptsetup uses: actions/checkout@v4 with: @@ -135,11 +128,21 @@ jobs: - name: Run cryptsetup tests working-directory: cryptsetup run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + + # from the cryptsetup source root + make -j$(nproc) + make -C tests check TESTS="vectors-test run-all-symbols unit-utils-crypt-test" VERBOSE=1 2>&1 | tee cryptsetup-test.log + TEST_RESULT=$(grep -q "All 3 tests passed" cryptsetup-test.log && echo "0" || echo "1") + printf "TEST_RESULT: $TEST_RESULT\n" + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" cryptsetup + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # from the cryptsetup source root make -j$(nproc) make -C tests check TESTS="vectors-test run-all-symbols unit-utils-crypt-test" VERBOSE=1 2>&1 | tee cryptsetup-test.log TEST_RESULT=$(grep -q "All 3 tests passed" cryptsetup-test.log && echo "0" || echo "1") printf "TEST_RESULT: $TEST_RESULT\n" - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} cryptsetup + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" cryptsetup diff --git a/.github/workflows/libeac3.yml b/.github/workflows/libeac3.yml index 3c53ff21..a494856e 100644 --- a/.github/workflows/libeac3.yml +++ b/.github/workflows/libeac3.yml @@ -1,19 +1,20 @@ name: libeac3 Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for libeac3 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_libeac3: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: openpace_ref: [ '1.1.3' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -86,12 +88,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install libeac3 dependencies - run: | - apt-get update - apt-get install -y autoconf automake libtool libc6 help2man gengetopt \ - pkg-config m4 patch autoconf automake libtool pkg-config build-essential - - name: Checkout openpace uses: actions/checkout@v4 with: @@ -125,7 +121,19 @@ jobs: - name: Run libeac3 tests working-directory: openpace run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + ./src/eactest > libeac3-test.log || echo "eactest failed with exit code $?" + cat libeac3-test.log + + if grep -q "Everything works as expected." libeac3-test.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" libeac3 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 ./src/eactest > libeac3-test.log || echo "eactest failed with exit code $?" cat libeac3-test.log @@ -134,4 +142,4 @@ jobs: else TEST_RESULT=1 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} libeac3 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" libeac3 diff --git a/.github/workflows/libfido2.yml b/.github/workflows/libfido2.yml index cf375313..f1f4a606 100644 --- a/.github/workflows/libfido2.yml +++ b/.github/workflows/libfido2.yml @@ -1,15 +1,20 @@ name: libfido2 Tests + +# OSP integration test for libfido2 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. + on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -17,27 +22,28 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_libfido2: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 15 strategy: + fail-fast: false matrix: libfido2_ref: [ '1.15.0' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -79,12 +85,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install test dependencies - run: | - apt-get update - apt-get install -y build-essential cmake pkg-config libudev-dev \ - zlib1g-dev libcbor-dev libpcsclite-dev pcscd - - name: Checkout libfido2 uses: actions/checkout@v4 with: @@ -124,7 +124,22 @@ jobs: - name: Run libfido2 tests working-directory: libfido2_repo/build run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run tests, excluding regress_dev which requires hardware/fails in CI + ctest --exclude-regex "regress_dev" 2>&1 | tee libfido2-test.log + + # Check test results directly in YAML + if grep -q "100% tests passed" libfido2-test.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" libfido2 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests, excluding regress_dev which requires hardware/fails in CI ctest --exclude-regex "regress_dev" 2>&1 | tee libfido2-test.log @@ -136,4 +151,4 @@ jobs: TEST_RESULT=1 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} libfido2 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" libfido2 diff --git a/.github/workflows/libhashkit2.yml b/.github/workflows/libhashkit2.yml index db5844c9..7d2218b3 100644 --- a/.github/workflows/libhashkit2.yml +++ b/.github/workflows/libhashkit2.yml @@ -1,19 +1,20 @@ name: libhashkit2 Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for libhashkit2 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_libhashkit2: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: libhashkit2_ref: [ '1.1.4' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,11 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install libmemcached dependencies - run: | - apt-get update - apt-get install -y cmake build-essential bison flex memcached libc6 - - name: Download libmemcached uses: actions/checkout@v4 with: @@ -109,7 +106,20 @@ jobs: - name: Run libhashkit2 tests working-directory: libmemcached/build run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + # Run tests + make test 2>&1 | tee libhashkit2-test.log + if grep -q "(Failed)" libhashkit2-test.log; then + TEST_RESULT=1 + else + TEST_RESULT=0 + fi + echo "TEST_RESULT = $TEST_RESULT" + + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" libhashkit2 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests make test 2>&1 | tee libhashkit2-test.log if grep -q "(Failed)" libhashkit2-test.log; then @@ -119,4 +129,4 @@ jobs: fi echo "TEST_RESULT = $TEST_RESULT" - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} libhashkit2 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" libhashkit2 diff --git a/.github/workflows/libnice.yml b/.github/workflows/libnice.yml index e82a4ee0..1399c62c 100644 --- a/.github/workflows/libnice.yml +++ b/.github/workflows/libnice.yml @@ -1,19 +1,20 @@ name: libnice Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for libnice Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,27 +22,28 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_libnice: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 20 strategy: + fail-fast: false matrix: libnice_ref: [ '0.1.21' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -119,11 +121,21 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from ninja test - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run tests and save output to test.log + ninja -C builddir test 2>&1 | tee libnice_test.log + + # Capture the test result using PIPESTATUS (Bash only) + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" libnice + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests and save output to test.log ninja -C builddir test 2>&1 | tee libnice_test.log # Capture the test result using PIPESTATUS (Bash only) TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} libnice + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" libnice diff --git a/.github/workflows/liboauth2.yml b/.github/workflows/liboauth2.yml index 6a294be5..074cd59e 100644 --- a/.github/workflows/liboauth2.yml +++ b/.github/workflows/liboauth2.yml @@ -1,19 +1,20 @@ name: liboauth2 Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for liboauth2 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,27 +22,28 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_liboauth2: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 20 strategy: + fail-fast: false matrix: liboauth2_ref: [ 'v1.4.5.4' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -83,13 +85,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install liboauth2 dependencies - run: | - apt-get update - apt-get install -y libcurl4-openssl-dev libjansson-dev \ - libcjose-dev pkg-config build-essential apache2-dev libhiredis-dev \ - libmemcached-dev autotools-dev autoconf automake libtool check patch - - name: Checkout OSP uses: actions/checkout@v4 with: @@ -120,10 +115,19 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Build and run tests + make check 2>&1 | tee liboauth2-test.log + # Capture the test result using PIPESTATUS (Bash only) + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" liboauth2 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Build and run tests make check 2>&1 | tee liboauth2-test.log # Capture the test result using PIPESTATUS (Bash only) TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} liboauth2 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" liboauth2 diff --git a/.github/workflows/librelp.yml b/.github/workflows/librelp.yml index 002c4fd8..937d5e53 100644 --- a/.github/workflows/librelp.yml +++ b/.github/workflows/librelp.yml @@ -1,19 +1,20 @@ name: librelp Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for librelp Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,43 +22,38 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_librelp: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # Run inside Debian Bookworm to match packaging environment container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: # Dont test osp master since it might be unstable librelp_ref: [ 'v1.12.0' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages steps: - - name: Install dependencies - run: | - apt-get update - apt-get install -y git build-essential autoconf automake \ - libtool pkg-config libgnutls28-dev net-tools iproute2 python3 \ - valgrind libtool-bin - - name: Checkout wolfProvider uses: actions/checkout@v4 with: @@ -111,7 +107,13 @@ jobs: - name: Run librelp tests working-directory: librelp run: | - ${{ matrix.force_fail }} make check 2>&1 | tee librelp-test.log + # --- normal mode --- + make check 2>&1 | tee librelp-test.log + TEST_RESULT=$(grep -q "# FAIL: 0" librelp-test.log && echo "0" || echo "1") + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" librelp + + # --- force-fail mode --- + make clean + WOLFPROV_FORCE_FAIL=1 make check 2>&1 | tee librelp-test.log TEST_RESULT=$(grep -q "# FAIL: 0" librelp-test.log && echo "0" || echo "1") - echo "TEST_RESULT = $TEST_RESULT" - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} librelp + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" librelp diff --git a/.github/workflows/libssh2.yml b/.github/workflows/libssh2.yml index f5c59177..837d043c 100644 --- a/.github/workflows/libssh2.yml +++ b/.github/workflows/libssh2.yml @@ -1,19 +1,20 @@ name: libssh2 Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for libssh2 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,27 +22,28 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_libssh2: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 20 strategy: + fail-fast: false matrix: libssh2_ref: [ 'libssh2-1.10.0' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -83,14 +85,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y git sudo build-essential autoconf automake \ - libtool pkg-config libjansson-dev check ca-certificates dpkg-dev \ - clang libc++-dev python3-impacket openssh-client openssh-server - - name: Download libssh2 uses: actions/checkout@v4 with: @@ -121,7 +115,18 @@ jobs: working-directory: libssh2 shell: bash run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + # Always continue on errors to ensure we show test results + set +e + + # Run the tests and capture the result + set -o pipefail + make check 2>&1 | tee libssh2-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" libssh2 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Always continue on errors to ensure we show test results set +e @@ -129,4 +134,4 @@ jobs: set -o pipefail make check 2>&1 | tee libssh2-test.log TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} libssh2 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" libssh2 diff --git a/.github/workflows/libtss2.yml b/.github/workflows/libtss2.yml index aaf434b0..507608c5 100644 --- a/.github/workflows/libtss2.yml +++ b/.github/workflows/libtss2.yml @@ -1,45 +1,46 @@ name: tpm2-tss Tests + +# OSP integration test for tpm2-tss Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. + on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + test_tpm2_tss: + needs: discover_versions runs-on: ubuntu-22.04 + container: + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm + env: + DEBIAN_FRONTEND: noninteractive timeout-minutes: 30 strategy: + fail-fast: false matrix: tpm2_tss_ref: [ '4.1.3'] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_latest_ref_array) }} replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages steps: - - name: Install test dependencies - run: | - sudo apt-get update - sudo apt-get install -y pkg-config libcunit1-dev autoconf-archive \ - gettext libcmocka-dev build-essential autoconf automake libtool \ - libjson-c-dev libcurl4-openssl-dev acl libusb-1.0-0-dev git \ - pkg-config uuid-dev - - # ensure libssl-dev is not installed - - name: Ensure libssl-dev is not installed + # tpm2-tss configure picks up the stock /usr/include/openssl/* + # otherwise, which conflicts with the wolfssl-built openssl this + # workflow links against. Remove libssl-dev from the container so + # configure finds only the in-tree headers. + - name: Remove stock libssl-dev run: | if dpkg -l | grep -q libssl-dev; then - echo "libssl-dev is installed, removing it to avoid conflicts" - sudo apt-get remove -y libssl-dev - else - echo "libssl-dev is not installed, no action needed" + apt-get remove -y libssl-dev fi - name: Checkout wolfProvider @@ -49,7 +50,9 @@ jobs: - name: Build wolfProvider run: | - OPENSSL_TAG=${{ matrix.openssl_ref }} WOLFSSL_TAG=${{ matrix.wolfssl_ref }} ./scripts/build-wolfprovider.sh + OPENSSL_TAG=${{ matrix.openssl_ref }} \ + WOLFSSL_TAG=${{ matrix.wolfssl_ref }} \ + ./scripts/build-wolfprovider.sh - name: Checkout tpm2-tss uses: actions/checkout@v4 @@ -83,12 +86,23 @@ jobs: working-directory: tpm2_tss_repo run: | source $GITHUB_WORKSPACE/scripts/env-setup - export ${{ matrix.force_fail }} + # --- normal mode --- + make check 2>&1 || true + if grep -q "FAIL: test/unit" test-suite.log; then + TEST_RESULT=1 + echo "Expected zero failures" + else + TEST_RESULT=0 + fi + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" tpm2-tss + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 make check 2>&1 || true - if $(grep -q "FAIL: test/unit" test-suite.log); then + if grep -q "FAIL: test/unit" test-suite.log; then TEST_RESULT=1 echo "Expected zero failures" else TEST_RESULT=0 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} tpm2-tss + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" tpm2-tss diff --git a/.github/workflows/libwebsockets.yml b/.github/workflows/libwebsockets.yml index c471fce2..adb2450c 100644 --- a/.github/workflows/libwebsockets.yml +++ b/.github/workflows/libwebsockets.yml @@ -1,19 +1,20 @@ name: libwebsockets Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for libwebsockets Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_libwebsockets: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: libwebsockets_ref: [ 'v4.3.3' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -55,11 +57,6 @@ jobs: with: fetch-depth: 1 - - name: Install libwebsockets dependencies - run: | - apt-get update - apt-get install -y libc6 libcap2 zlib1g cmake build-essential dpkg-dev - - name: Download packages from build job uses: actions/download-artifact@v4 with: @@ -103,7 +100,26 @@ jobs: working-directory: libwebsockets shell: bash run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + + ./build/bin/libwebsockets-test-server --port=11111 --ssl > server.log 2>&1 & SERVER_PID=$! + sleep 5 + timeout 10 ./build/bin/libwebsockets-test-client 127.0.0.1 --port=11111 --ssl > client.log 2>&1 || echo "Client exited with error $?" + ldd ./build/bin/libwebsockets-test-server | grep wolfProvider || echo "wolfProvider not found in server" + ldd ./build/bin/libwebsockets-test-client | grep wolfProvider || echo "wolfProvider not found in client" + kill $SERVER_PID || echo "Server already exited" + cat server.log || echo "Missing server.log" + cat client.log || echo "Missing client.log" + cat server.log client.log > libwebsockets-test.log + if grep -q "error:03080006" libwebsockets-test.log || grep -q "Failed to create default vhost" libwebsockets-test.log; then + TEST_RESULT=1 + else + TEST_RESULT=0 + fi + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" libwebsockets + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 ./build/bin/libwebsockets-test-server --port=11111 --ssl > server.log 2>&1 & SERVER_PID=$! sleep 5 @@ -119,4 +135,4 @@ jobs: else TEST_RESULT=0 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} libwebsockets + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" libwebsockets diff --git a/.github/workflows/multi-compiler.yml b/.github/workflows/multi-compiler.yml index 8619977f..366e8ba5 100644 --- a/.github/workflows/multi-compiler.yml +++ b/.github/workflows/multi-compiler.yml @@ -6,6 +6,18 @@ on: branches: [ 'master', 'main', 'release/**' ] pull_request: branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -14,6 +26,7 @@ concurrency: jobs: build_wolfprovider: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false name: Build with compiler ${{ matrix.CC }}, wolfssl ${{ matrix.wolfssl_ref }}, OpenSSL ${{ matrix.openssl_ref }} runs-on: ${{ matrix.OS }} timeout-minutes: 20 diff --git a/.github/workflows/net-snmp.yml b/.github/workflows/net-snmp.yml index dcb806b3..a1ad22ca 100644 --- a/.github/workflows/net-snmp.yml +++ b/.github/workflows/net-snmp.yml @@ -1,19 +1,20 @@ name: Net-SNMP Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Net-SNMP Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,17 +22,18 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_net_snmp: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. @@ -40,10 +42,9 @@ jobs: fail-fast: false matrix: net_snmp_ref: [ 'v5.9.3' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -85,12 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y libperl-dev build-essential autoconf \ - libtool pkg-config gettext net-tools - - name: Checkout net-snmp uses: actions/checkout@v4 with: @@ -138,10 +133,19 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make test - export ${{ matrix.force_fail }} + # --- normal mode --- + + autoconf --version | grep -P '2\.\d\d' -o > dist/autoconf-version + make -j test TESTOPTS="-e agentxperl" 2>&1 | tee net-snmp-test.log + # Capture the test result using PIPESTATUS (Bash only) + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" net-snmp + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 autoconf --version | grep -P '2\.\d\d' -o > dist/autoconf-version make -j test TESTOPTS="-e agentxperl" 2>&1 | tee net-snmp-test.log # Capture the test result using PIPESTATUS (Bash only) TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} net-snmp + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" net-snmp diff --git a/.github/workflows/nginx.yml b/.github/workflows/nginx.yml index 1159b765..53e868b2 100644 --- a/.github/workflows/nginx.yml +++ b/.github/workflows/nginx.yml @@ -1,19 +1,20 @@ name: Nginx Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Nginx Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_nginx: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: nginx_ref: [ 'release-1.27.4' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,13 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update && \ - apt-get install -y perl build-essential autoconf automake libtool \ - pkg-config libpcre3-dev zlib1g-dev - cpan -iT Proc::Find Net::SSLeay IO::Socket::SSL - - name: Checkout nginx uses: actions/checkout@v4 with: @@ -116,9 +111,17 @@ jobs: - name: Run nginx-tests with wolfProvider working-directory: nginx-tests run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run tests and save result + TEST_NGINX_VERBOSE=y TEST_NGINX_CATLOG=y TEST_NGINX_BINARY=../nginx/objs/nginx prove -v . 2>&1 | tee nginx-test.log + TEST_RESULT=$? + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" nginx + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests and save result TEST_NGINX_VERBOSE=y TEST_NGINX_CATLOG=y TEST_NGINX_BINARY=../nginx/objs/nginx prove -v . 2>&1 | tee nginx-test.log TEST_RESULT=$? - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} nginx + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" nginx diff --git a/.github/workflows/nightly-osp.yml b/.github/workflows/nightly-osp.yml new file mode 100644 index 00000000..38d69c74 --- /dev/null +++ b/.github/workflows/nightly-osp.yml @@ -0,0 +1,235 @@ +name: Nightly OSP Suite + +# Nightly orchestrator that runs every OSP (Open Source Project) +# integration workflow against the latest stable wolfSSL and the +# current Debian Bookworm OpenSSL. Each OSP workflow is a reusable +# (workflow_call) workflow; this file just fans them out in parallel +# and reports an aggregate status to Slack. +# +# Triggers: +# - schedule: daily at 06:00 UTC +# - workflow_dispatch: manual fire from the Actions UI +# - pull_request: TEMPORARY -- so PR #400's reviewers can see +# the dispatcher actually fan everything out +# and the notify job hit Slack. Revert this +# trigger before merging if you don't want +# nightly's full job set firing on every PR. + +on: + schedule: + # 06:00 UTC daily. Pick a time when shared-runner contention is low + # so the matrix doesn't fight PR CI. + - cron: '0 6 * * *' + pull_request: + branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' + workflow_dispatch: + inputs: + reason: + description: 'Why are we triggering manually? (annotation only)' + required: false + default: 'manual nightly run' + +permissions: + contents: read + packages: read + actions: read + +concurrency: + group: nightly-osp + cancel-in-progress: false + +jobs: + # Each entry below fires the corresponding OSP workflow as a reusable. + # Adding a new OSP workflow? Drop a `:` block here too. + bind9: { uses: ./.github/workflows/bind9.yml } + cjose: { uses: ./.github/workflows/cjose.yml } + curl: { uses: ./.github/workflows/curl.yml } + debian-package: { uses: ./.github/workflows/debian-package.yml } + git-ssh-dr: { uses: ./.github/workflows/git-ssh-dr.yml } + grpc: { uses: ./.github/workflows/grpc.yml } + hostap: { uses: ./.github/workflows/hostap.yml } + iperf: { uses: ./.github/workflows/iperf.yml } + krb5: { uses: ./.github/workflows/krb5.yml } + libcryptsetup: { uses: ./.github/workflows/libcryptsetup.yml } + libeac3: { uses: ./.github/workflows/libeac3.yml } + libfido2: { uses: ./.github/workflows/libfido2.yml } + libhashkit2: { uses: ./.github/workflows/libhashkit2.yml } + libnice: { uses: ./.github/workflows/libnice.yml } + liboauth2: { uses: ./.github/workflows/liboauth2.yml } + librelp: { uses: ./.github/workflows/librelp.yml } + libssh2: { uses: ./.github/workflows/libssh2.yml } + libtss2: { uses: ./.github/workflows/libtss2.yml } + libwebsockets: { uses: ./.github/workflows/libwebsockets.yml } + net-snmp: { uses: ./.github/workflows/net-snmp.yml } + nginx: { uses: ./.github/workflows/nginx.yml } + openldap: { uses: ./.github/workflows/openldap.yml } + opensc: { uses: ./.github/workflows/opensc.yml } + openssh: { uses: ./.github/workflows/openssh.yml } + openvpn: { uses: ./.github/workflows/openvpn.yml } + pam-pkcs11: { uses: ./.github/workflows/pam-pkcs11.yml } + ppp: { uses: ./.github/workflows/ppp.yml } + python3-ntp: { uses: ./.github/workflows/python3-ntp.yml } + qt5network5: { uses: ./.github/workflows/qt5network5.yml } + rsync: { uses: ./.github/workflows/rsync.yml } + socat: { uses: ./.github/workflows/socat.yml } + sscep: { uses: ./.github/workflows/sscep.yml } + sssd: { uses: ./.github/workflows/sssd.yml } + stunnel: { uses: ./.github/workflows/stunnel.yml } + systemd: { uses: ./.github/workflows/systemd.yml } + tcpdump: { uses: ./.github/workflows/tcpdump.yml } + tnftp: { uses: ./.github/workflows/tnftp.yml } + tpm2-tools: { uses: ./.github/workflows/tpm2-tools.yml } + x11vnc: { uses: ./.github/workflows/x11vnc.yml } + xmlsec: { uses: ./.github/workflows/xmlsec.yml } + # Internal sweep: every upstream openssl-3.X.Y release tag. + openssl-version: { uses: ./.github/workflows/openssl-version.yml } + + notify: + name: Slack notification + needs: + - bind9 + - cjose + - curl + - debian-package + - git-ssh-dr + - grpc + - hostap + - iperf + - krb5 + - libcryptsetup + - libeac3 + - libfido2 + - libhashkit2 + - libnice + - liboauth2 + - librelp + - libssh2 + - libtss2 + - libwebsockets + - net-snmp + - nginx + - openldap + - opensc + - openssh + - openvpn + - pam-pkcs11 + - ppp + - python3-ntp + - qt5network5 + - rsync + - socat + - sscep + - sssd + - stunnel + - systemd + - tcpdump + - tnftp + - tpm2-tools + - x11vnc + - xmlsec + - openssl-version + if: always() + runs-on: ubuntu-latest + # Job-level env so step `if:` blocks can see SLACK_WEBHOOK_URL -- + # step-level `env:` is not available to the same step's `if:` + # because the expression is evaluated before the env mapping is + # exported. Setting at the job level avoids that ordering trap. + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + steps: + - name: Compose summary + id: summary + env: + NEEDS_JSON: ${{ toJSON(needs) }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + # Pass/fail lists. Use `to_entries[]` (stream) so each result + # is its own object inside the string interpolation -- doing + # this inside a `map(...)` would make .[].key stream multiple + # values into a single template slot, which is malformed jq. + PASSED=$(echo "$NEEDS_JSON" | jq -r ' + to_entries[] + | select(.value.result == "success") + | .key' | sort) + FAILED=$(echo "$NEEDS_JSON" | jq -r ' + to_entries[] + | select(.value.result != "success") + | "\(.key) (\(.value.result))"' | sort) + PASS_COUNT=$(echo "$PASSED" | grep -c . || echo 0) + FAIL_COUNT=$(echo "$FAILED" | grep -c . || echo 0) + TOTAL=$((PASS_COUNT + FAIL_COUNT)) + + if [ "$FAIL_COUNT" -eq 0 ]; then + COLOR="good" + TITLE=":large_green_circle: Nightly OSP: ALL ${TOTAL} PASSED" + DETAILS="All OSP workflows green for $(date -u +%Y-%m-%d)." + else + COLOR="danger" + TITLE=":red_circle: Nightly OSP: ${FAIL_COUNT} of ${TOTAL} FAILED" + DETAILS=$'Failed:\n```\n'"${FAILED}"$'\n```' + fi + + { + echo "color=$COLOR" + echo "title<> "$GITHUB_OUTPUT" + + - name: Post to Slack + # Skip silently when the webhook isn't configured (forks, manual + # dispatch by someone without the secret, etc.). The `env` is + # job-level above, which is what makes this `if:` work. + if: env.SLACK_WEBHOOK_URL != '' + env: + TITLE: ${{ steps.summary.outputs.title }} + DETAILS: ${{ steps.summary.outputs.details }} + COLOR: ${{ steps.summary.outputs.color }} + run: | + set -euo pipefail + PAYLOAD=$(jq -n \ + --arg title "$TITLE" \ + --arg details "$DETAILS" \ + --arg color "$COLOR" \ + '{ + attachments: [{ + color: $color, + title: $title, + text: $details, + mrkdwn_in: ["text"] + }] + }') + # -f makes curl exit non-zero on HTTP >= 400 so the workflow + # actually fails if Slack rejects the payload. + curl -fsS -X POST -H 'Content-type: application/json' \ + --data "$PAYLOAD" "$SLACK_WEBHOOK_URL" >/dev/null + + - name: Log to job output (always) + # So you can read the same summary in the GitHub UI even when + # Slack isn't configured. + env: + TITLE: ${{ steps.summary.outputs.title }} + DETAILS: ${{ steps.summary.outputs.details }} + run: | + { + echo "## $TITLE" + echo "" + echo "$DETAILS" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/openldap.yml b/.github/workflows/openldap.yml index 5b85854a..102a93cc 100644 --- a/.github/workflows/openldap.yml +++ b/.github/workflows/openldap.yml @@ -1,19 +1,20 @@ name: OpenLDAP Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for OpenLDAP Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,17 +22,18 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_openldap: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. @@ -40,11 +42,11 @@ jobs: fail-fast: false matrix: openldap_ref: [ 'OPENLDAP_REL_ENG_2_6_7' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -85,14 +87,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y git sudo build-essential autoconf automake \ - libtool pkg-config libjansson-dev check ca-certificates dpkg-dev \ - groff libsasl2-dev - - name: Checkout openldap uses: actions/checkout@v4 with: @@ -128,22 +122,21 @@ jobs: make -j depend make -j - export ${{ matrix.force_fail }} - if [ -n "${{ matrix.force_fail }}" ]; then - set +e - fi - - if [ "${{ matrix.force_fail }}" = "WOLFPROV_FORCE_FAIL=1" ]; then - # Run with a 15 minute timeout for WPFF since it breaks on test 067 - timeout 15m make -j check 2>&1 | tee openldap-test.log - TEST_RESULT=${PIPESTATUS[0]} - if [ $TEST_RESULT -eq 124 ]; then - echo "make -j check timed out after 15 minutes with WOLFPROV_FORCE_FAIL=1" - echo "Tests failed to complete as expected" - TEST_RESULT=1 - fi - else - make -j check 2>&1 | tee openldap-test.log - TEST_RESULT=${PIPESTATUS[0]} + # --- normal mode --- + make -j check 2>&1 | tee openldap-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" openldap + + # --- force-fail mode --- + # WPFF breaks on test 067; cap at 15 min and treat timeout as failure. + export WOLFPROV_FORCE_FAIL=1 + set +e + timeout 15m make -j check 2>&1 | tee openldap-test.log + TEST_RESULT=${PIPESTATUS[0]} + set -e + if [ $TEST_RESULT -eq 124 ]; then + echo "make -j check timed out after 15 minutes with WOLFPROV_FORCE_FAIL=1" + echo "Tests failed to complete as expected" + TEST_RESULT=1 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} openldap + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" openldap diff --git a/.github/workflows/opensc.yml b/.github/workflows/opensc.yml index f8b44d12..898f53cc 100644 --- a/.github/workflows/opensc.yml +++ b/.github/workflows/opensc.yml @@ -1,19 +1,20 @@ name: OpenSC Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for OpenSC Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_opensc: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 30 strategy: + fail-fast: false matrix: opensc_ref: [ '0.25.1' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,15 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install OpenSC dependencies - run: | - apt-get update - apt-get install -y \ - autotools-dev libtool automake autoconf make pkg-config \ - libeac-dev gengetopt libpcsclite-dev libreadline-dev \ - zlib1g-dev docbook-xsl xsltproc pcscd softhsm2 opensc pcsc-tools \ - vim libcmocka-dev libjson-c-dev libp11-dev patch - - name: Download OpenSC uses: actions/checkout@v4 with: @@ -133,11 +126,25 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} - # Run tests and save output + check_opensc_log() { + # 18 passes, 2 expected failures expected. + if grep -q "# PASS: 10" opensc-test.log \ + && grep -q "# PASS: 8" opensc-test.log \ + && grep -q "# XFAIL: 2" opensc-test.log; then + echo 0 + else + echo 1 + fi + } + + # --- normal mode --- make check | tee opensc-test.log + TEST_RESULT=$(check_opensc_log) + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" opensc - # Check for expected test results in the test log (18 passes, 2 expected failures, with WPFF we expect 6 failures) - TEST_RESULT=$(((grep -q "# PASS: 10" opensc-test.log) && (grep -q "# PASS: 8" opensc-test.log) && (grep -q "# XFAIL: 2" opensc-test.log)) && echo "0" || echo "1") - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} opensc + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + make check | tee opensc-test.log + TEST_RESULT=$(check_opensc_log) + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" opensc diff --git a/.github/workflows/openssh.yml b/.github/workflows/openssh.yml index b4b2e835..c1126003 100644 --- a/.github/workflows/openssh.yml +++ b/.github/workflows/openssh.yml @@ -1,19 +1,20 @@ name: openssh Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for openssh Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,17 +22,18 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_openssh: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm # Extra permissions needed for Debian Bookworm options: >- --privileged @@ -47,12 +49,12 @@ jobs: # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: openssh_ref: [ 'V_10_0_P2', 'V_9_9_P1' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'non-FIPS' ] # FIPS is not yet supported for OpenSSH - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -94,12 +96,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y build-essential autoconf automake libtool \ - pkg-config patch zlib1g-dev kmod util-linux cryptsetup-bin - - name: Ensure kernel modules are present run: | # loop + device-mapper (dm-crypt); scsi_debug is optional and may still be unavailable on the host kernel @@ -134,7 +130,46 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Enable unsafe permissions for testing + export TEST_SSH_UNSAFE_PERMISSIONS=1 + + # Priv-sep user/group (idempotent) + getent group sshd >/dev/null || addgroup --system sshd + id -u sshd >/dev/null 2>&1 || adduser --system --no-create-home \ + --ingroup sshd --home /nonexistent --shell /usr/sbin/nologin sshd + + # Priv-sep runtime dirs + install -d -m 0755 /run/sshd + + # The required chroot for privilege separation + # Must exist, be owned by root, and not be writable by group/world. + install -d -o root -g root -m 0755 /var/empty + + # Ensure the privsep user/group exist (idempotent) + if ! getent group sshd >/dev/null; then + addgroup --system sshd + fi + if ! id -u sshd >/dev/null 2>&1; then + adduser --system --no-create-home --ingroup sshd \ + --home /nonexistent --shell /usr/sbin/nologin sshd + fi + + autoreconf -ivf + ./configure --with-prngd-socket=/tmp/prngd \ + --with-ldflags=-Wl,--export-dynamic + make -j + + export LD_LIBRARY_PATH=".:openbsd-compat:$LD_LIBRARY_PATH" # Include build dirs for symbol resolution + + # Run all the tests except (t-exec) as it takes too long + make file-tests interop-tests extra-tests unit 2>&1 | tee openssh-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" openssh + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Enable unsafe permissions for testing export TEST_SSH_UNSAFE_PERMISSIONS=1 @@ -170,4 +205,4 @@ jobs: # Run all the tests except (t-exec) as it takes too long make file-tests interop-tests extra-tests unit 2>&1 | tee openssh-test.log TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} openssh + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" openssh diff --git a/.github/workflows/openssl-version.yml b/.github/workflows/openssl-version.yml index 90e6a77d..a3b0c514 100644 --- a/.github/workflows/openssl-version.yml +++ b/.github/workflows/openssl-version.yml @@ -1,74 +1,36 @@ name: OpenSSL Version Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# Full sweep across every upstream openssl-3.X.Y release tag, resolved +# at run time by _discover-versions.yml. ~58 versions x 2 wolfssl refs +# = ~116 jobs, which is nightly-shaped work, not per-PR. +# +# Runs nightly via the Nightly OSP Suite orchestrator +# (.github/workflows/nightly-osp.yml) or manually via workflow_dispatch. +# PR-side OpenSSL coverage comes from simple.yml (newest upstream + +# oldest 3.0.x LTS, against master + latest-stable wolfssl). -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + openssl_version_test: + needs: discover_versions continue-on-error: true name: OpenSSL Version Test runs-on: ubuntu-22.04 timeout-minutes: 30 strategy: + fail-fast: false matrix: - wolfssl_ref: ['v5.8.4-stable'] - openssl_ref: [ - 'openssl-3.0.3', - 'openssl-3.0.5', - 'openssl-3.0.6', - 'openssl-3.0.7', - 'openssl-3.0.8', - 'openssl-3.0.9', - 'openssl-3.0.10', - 'openssl-3.0.11', - 'openssl-3.0.12', - 'openssl-3.0.13', - 'openssl-3.0.14', - 'openssl-3.0.15', - 'openssl-3.0.16', - 'openssl-3.0.17', - 'openssl-3.0.18', - 'openssl-3.1.0', - 'openssl-3.1.1', - 'openssl-3.1.2', - 'openssl-3.1.3', - 'openssl-3.1.4', - 'openssl-3.1.5', - 'openssl-3.1.6', - 'openssl-3.1.7', - 'openssl-3.1.8', - 'openssl-3.2.0', - 'openssl-3.2.1', - 'openssl-3.2.2', - 'openssl-3.2.3', - 'openssl-3.2.4', - 'openssl-3.2.5', - 'openssl-3.2.6', - 'openssl-3.3.0', - 'openssl-3.3.1', - 'openssl-3.3.2', - 'openssl-3.3.3', - 'openssl-3.3.4', - 'openssl-3.3.5', - 'openssl-3.4.0', - 'openssl-3.4.1', - 'openssl-3.4.2', - 'openssl-3.4.3', - 'openssl-3.5.0', - 'openssl-3.5.1', - 'openssl-3.5.2', - 'openssl-3.5.3', - 'openssl-3.5.4', - 'openssl-3.6.0'] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + # Every upstream openssl-3.X.Y release tag, resolved at run time + # by _discover-versions.yml. New release ships -> next run picks + # it up automatically; no PR needed to track the latest patch. + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_all_releases_array) }} steps: - name: Checkout wolfProvider uses: actions/checkout@v4 diff --git a/.github/workflows/openvpn.yml b/.github/workflows/openvpn.yml index de421158..8b2522d1 100644 --- a/.github/workflows/openvpn.yml +++ b/.github/workflows/openvpn.yml @@ -1,19 +1,20 @@ name: OpenVPN Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for OpenVPN Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,19 +22,20 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_openvpn: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # This should be a safe limit for the tests to run. timeout-minutes: 20 container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive strategy: @@ -41,11 +43,11 @@ jobs: matrix: # Dont test master since it might be too unstable openvpn_ref: [ 'v2.6.12' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -87,17 +89,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Set up environment - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y git sudo build-essential autoconf automake \ - libtool pkg-config libjansson-dev check ca-certificates dpkg-dev \ - liblzo2-dev libpam0g-dev liblz4-dev libcap-ng-dev \ - linux-libc-dev man2html libcmocka-dev python3-docutils \ - iproute2 libtool automake autoconf libnl-genl-3-dev \ - libnl-genl-3-200 - - name: Find ossl headers run: | find / -name ssl.h 2>/dev/null || true @@ -132,13 +123,16 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} - if [ -n "${{ matrix.force_fail }}" ]; then - set +e - fi - # Run tests and save result + # --- normal mode --- + make check 2>&1 | tee openvpn-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" openvpn + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + set +e make check 2>&1 | tee openvpn-test.log - # Capture the test result using PIPESTATUS (Bash only) TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} openvpn + set -e + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" openvpn diff --git a/.github/workflows/pam-pkcs11.yml b/.github/workflows/pam-pkcs11.yml index a3666bba..ff574dd3 100644 --- a/.github/workflows/pam-pkcs11.yml +++ b/.github/workflows/pam-pkcs11.yml @@ -1,19 +1,20 @@ name: pam-pkcs11 Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for pam-pkcs11 Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_pam_pkcs11: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: pam_pkcs11_ref: [ 'pam_pkcs11-0.6.12' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -54,12 +56,6 @@ jobs: with: fetch-depth: 1 - - name: Install git and basic dependencies - run: | - apt-get update - apt-get install -y git - - # Avoid "detected dubious ownership" warning - name: Ensure the working directory safe run: | git config --global --add safe.directory "$GITHUB_WORKSPACE" @@ -94,16 +90,27 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install test dependencies - run: | - apt-get update - apt-get install -y pkg-config build-essential autoconf automake libtool - - name: Run pam_pkcs11 tests shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- + export PAM_PKCS11_REF=${{ matrix.pam_pkcs11_ref }} + + # Run tests + if timeout 300 $GITHUB_WORKSPACE/.github/scripts/pam-pkcs11-test.sh; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + + echo "TEST_RESULT: $TEST_RESULT" + + # Capture result + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" pam_pkcs11 + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 export PAM_PKCS11_REF=${{ matrix.pam_pkcs11_ref }} # Run tests @@ -116,4 +123,4 @@ jobs: echo "TEST_RESULT: $TEST_RESULT" # Capture result - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} pam_pkcs11 + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" pam_pkcs11 diff --git a/.github/workflows/ppp.yml b/.github/workflows/ppp.yml index 457f81c5..4360859a 100644 --- a/.github/workflows/ppp.yml +++ b/.github/workflows/ppp.yml @@ -1,17 +1,20 @@ name: PPP Tests -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for PPP Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -19,30 +22,31 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_ppp: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 15 strategy: + fail-fast: false matrix: # Switched to v2.5.2 due to significant limitations with v2.4.9, # specifically the lack of a test suite, necessary configure options, # and compatibility with newer versions of openssl ppp_ref: [ 'v2.5.2' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -86,11 +90,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y build-essential autoconf libtool patch - - name: Checkout PPP uses: actions/checkout@v4 with: @@ -134,7 +133,22 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run tests + make check 2>&1 | tee ppp-test.log + + # Check test results directly in YAML + if grep -q "# FAIL: 0" pppd/test-suite.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" ppp + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests make check 2>&1 | tee ppp-test.log @@ -146,4 +160,4 @@ jobs: TEST_RESULT=1 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} ppp + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" ppp diff --git a/.github/workflows/publish-test-deps-image.yml b/.github/workflows/publish-test-deps-image.yml new file mode 100644 index 00000000..36f7c3b4 --- /dev/null +++ b/.github/workflows/publish-test-deps-image.yml @@ -0,0 +1,59 @@ +name: Publish test-deps image + +# Builds docker/wolfprovider-test-deps/Dockerfile and pushes it to +# ghcr.io/wolfssl/wolfprovider-test-deps:bookworm. +# +# Fires when the Dockerfile (or this workflow file) changes on master. +# The pushed package stays private -- consumer workflows running on +# wolfSSL/wolfProvider use the canonical GITHUB_TOKEN, which has read +# access to the org's private packages. + +on: + push: + branches: [ 'master', 'main' ] + paths: + - 'docker/wolfprovider-test-deps/**' + - '.github/workflows/publish-test-deps-image.yml' + workflow_dispatch: {} + +concurrency: + group: publish-test-deps-image + cancel-in-progress: false + +permissions: + contents: read + packages: write + +jobs: + publish: + # Only the canonical repo's runner has a token authorized to push + # to ghcr.io/wolfssl/*. Forks won't have that scope, so skip. + if: github.repository == 'wolfSSL/wolfProvider' + runs-on: ubuntu-22.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to ghcr.io + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push image + uses: docker/build-push-action@v6 + with: + context: docker/wolfprovider-test-deps + file: docker/wolfprovider-test-deps/Dockerfile + push: true + tags: | + ghcr.io/wolfssl/wolfprovider-test-deps:bookworm + ghcr.io/wolfssl/wolfprovider-test-deps:bookworm-${{ github.sha }} + cache-from: type=registry,ref=ghcr.io/wolfssl/wolfprovider-test-deps:bookworm + cache-to: type=inline diff --git a/.github/workflows/python3-ntp.yml b/.github/workflows/python3-ntp.yml index 21881f32..865a3e96 100644 --- a/.github/workflows/python3-ntp.yml +++ b/.github/workflows/python3-ntp.yml @@ -1,19 +1,20 @@ name: python3-ntp Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for python3-ntp Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,19 +22,20 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_python3-ntp: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # This should be a safe limit for the tests to run. timeout-minutes: 20 container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm options: --user root env: DEBIAN_FRONTEND: noninteractive @@ -41,12 +43,12 @@ jobs: OPENSSL_PACKAGES_PATH: /tmp/openssl-packages WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages strategy: + fail-fast: false matrix: python3-ntp_ref: [ 'NTPsec_1_2_2' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] steps: - name: Checkout wolfProvider @@ -84,12 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install python3-ntp dependencies - run: | - apt-get update - apt-get install -y build-essential bison libcap-dev libseccomp-dev \ - libavahi-compat-libdnssd-dev pps-tools python-dev-is-python3 - - name: Checkout python3-ntp uses: actions/checkout@v4 with: @@ -117,7 +113,19 @@ jobs: - name: Run python3-ntp tests working-directory: ntpsec run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run tests + ./waf check | tee python3-ntp-test.log + if grep -q "'check' finished successfully" python3-ntp-test.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" python3-ntp + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests ./waf check | tee python3-ntp-test.log @@ -126,4 +134,4 @@ jobs: else TEST_RESULT=1 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} python3-ntp + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" python3-ntp diff --git a/.github/workflows/qt5network5.yml b/.github/workflows/qt5network5.yml index f12581d0..6dc1a6d5 100644 --- a/.github/workflows/qt5network5.yml +++ b/.github/workflows/qt5network5.yml @@ -1,16 +1,20 @@ name: qtbase Network Tests -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# OSP integration test for qtbase Network Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. + +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -18,27 +22,28 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_qtbase_network: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 40 strategy: + fail-fast: false matrix: qt_ref: [ 'v5.15.8-lts-lgpl' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -80,13 +85,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install Qt dependencies - run: | - apt-get update - apt-get install -y build-essential pkg-config dpkg-dev \ - python3 perl libpcre2-dev zlib1g-dev cmake ninja-build \ - bison flex libpng-dev libjpeg-dev git ca-certificates - - name: Checkout OSP uses: actions/checkout@v4 with: @@ -130,7 +128,24 @@ jobs: shell: bash run: | set +e - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run the QSSLSocket test, the make check takes too long + QTEST_ENVIRONMENT=ci ./tests/auto/network/ssl/qsslsocket/tst_qsslsocket 2>&1 | tee qsslsocket-test.log + + # Check test results based on qt_ref + if grep -q "521 passed" qsslsocket-test.log; then + TEST_RESULT=0 + echo "SUCCESS: Found 521 passed tests as expected" + else + TEST_RESULT=1 + echo "Tests failed unexpectedly for 'v5.15.8-lts-lgpl' branch." + fi + + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" qtbase-qsslsocket + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run the QSSLSocket test, the make check takes too long QTEST_ENVIRONMENT=ci ./tests/auto/network/ssl/qsslsocket/tst_qsslsocket 2>&1 | tee qsslsocket-test.log @@ -144,4 +159,4 @@ jobs: echo "Tests failed unexpectedly for 'v5.15.8-lts-lgpl' branch." fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} qtbase-qsslsocket + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" qtbase-qsslsocket diff --git a/.github/workflows/rsync.yml b/.github/workflows/rsync.yml index 57f64e20..ba22160e 100644 --- a/.github/workflows/rsync.yml +++ b/.github/workflows/rsync.yml @@ -1,16 +1,20 @@ name: rsync Tests -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# OSP integration test for rsync Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. + +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -18,18 +22,19 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_rsync: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] timeout-minutes: 15 container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm options: --user root env: DEBIAN_FRONTEND: noninteractive @@ -37,12 +42,12 @@ jobs: OPENSSL_PACKAGES_PATH: /tmp/openssl-packages WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages strategy: + fail-fast: false matrix: rsync_ref: [ 'v3.2.7' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] steps: - name: Checkout wolfProvider @@ -80,13 +85,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install rsync dependencies - run: | - apt-get update - apt-get install -y gcc g++ gawk autoconf automake python3-cmarkgfm \ - acl libacl1-dev attr libattr1-dev libxxhash-dev \ - libzstd-dev liblz4-dev build-essential - - name: Checkout rsync uses: actions/checkout@v4 with: @@ -111,7 +109,22 @@ jobs: - name: Run rsync tests working-directory: rsync_repo run: | - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run rsync test suite including our SHA test + make check 2>&1 | tee rsync-test.log + + # Check test results - look for "0 failed" in the output + if grep -q "overall result is 0" rsync-test.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" rsync + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run rsync test suite including our SHA test make check 2>&1 | tee rsync-test.log @@ -123,4 +136,4 @@ jobs: TEST_RESULT=1 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} rsync + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" rsync diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml new file mode 100644 index 00000000..b61274dc --- /dev/null +++ b/.github/workflows/sanitizers.yml @@ -0,0 +1,107 @@ +name: Sanitizers (ASan + UBSan) + +# Build wolfSSL, OpenSSL, and wolfProvider from source with +# -fsanitize=address,undefined, then run the cmd-test suite under the +# instrumented binaries. Catches use-after-free, double-free, OOB +# read/write, and undefined behavior (signed overflow, alignment, +# misuse of bool, etc.). +# +# Static libasan is used so the wolfProvider .so picks it up without +# the test runner needing LD_PRELOAD. Detect-leaks stays on to surface +# anything we're forgetting to free. + +# START OF COMMON SECTION +on: + push: + branches: [ 'master', 'main', 'release/**' ] + pull_request: + branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +# END OF COMMON SECTION + +jobs: + discover_versions: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + uses: ./.github/workflows/_discover-versions.yml + + sanitizers: + needs: discover_versions + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + name: ASan+UBSan (wolfSSL ${{ matrix.wolfssl_ref }} / ${{ needs.discover_versions.outputs.openssl_latest_ref }}) + runs-on: ubuntu-22.04 + # Sanitizers add ~2-3x to build/test time vs. a plain build. + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + # Test master + latest-stable (resolved at run time). + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + env: + # Surface every report. halt_on_error=1 fails the first time we + # touch UB so we don't drown in cascades. + ASAN_OPTIONS: detect_leaks=1:halt_on_error=1:abort_on_error=1:print_stacktrace=1 + UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1 + steps: + - name: Checkout wolfProvider + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Install host build deps + # Bare runner -- not the test-deps container -- because we're + # building OpenSSL/wolfssl from source against the host toolchain. + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential autoconf automake libtool pkg-config \ + git curl wget patch m4 gettext + + - name: Build wolfProvider with sanitizers + env: + # Static libasan so the wolfProvider .so embeds it; otherwise + # the runtime needs LD_PRELOAD and ordering issues bite. + SAN_FLAGS: "-fsanitize=address,undefined -fno-omit-frame-pointer -fno-sanitize-recover=all -g" + SAN_LDFLAGS: "-fsanitize=address,undefined -static-libasan" + OPENSSL_CFLAGS: "-fsanitize=address,undefined -fno-omit-frame-pointer -fno-sanitize-recover=all -g -static-libasan" + OPENSSL_CXXFLAGS: "-fsanitize=address,undefined -fno-omit-frame-pointer -fno-sanitize-recover=all -g -static-libasan" + OPENSSL_LDFLAGS: "-fsanitize=address,undefined -static-libasan" + run: | + # wolfSSL and wolfProvider build scripts accept _CONFIG_CFLAGS + # via env; append the sanitizer flags so all three layers + # (OpenSSL, wolfSSL, wolfProvider) get instrumented. + export WOLFSSL_CONFIG_CFLAGS="${WOLFSSL_CONFIG_CFLAGS:-} ${SAN_FLAGS}" + export WOLFPROV_CONFIG_CFLAGS="${WOLFPROV_CONFIG_CFLAGS:-} ${SAN_FLAGS}" + + OPENSSL_TAG=${{ needs.discover_versions.outputs.openssl_latest_ref }} \ + WOLFSSL_TAG=${{ matrix.wolfssl_ref }} \ + ./scripts/build-wolfprovider.sh + + - name: Run cmd-tests under sanitizers + run: | + source scripts/env-setup + ./scripts/cmd_test/do-cmd-tests.sh + + - name: Dump build/test logs on failure + if: ${{ failure() }} + run: | + for f in test-suite.log scripts/build-release.log scripts/build-debug.log; do + if [ -f "$f" ]; then + echo "=== $f (last 200 lines) ===" + tail -200 "$f" + fi + done diff --git a/.github/workflows/seed-src.yml b/.github/workflows/seed-src.yml index 37e89703..ef21bbeb 100644 --- a/.github/workflows/seed-src.yml +++ b/.github/workflows/seed-src.yml @@ -6,6 +6,18 @@ on: branches: [ 'master', 'main', 'release/**' ] pull_request: branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -13,18 +25,23 @@ concurrency: # END OF COMMON SECTION jobs: + discover_versions: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + uses: ./.github/workflows/_discover-versions.yml + seed_src_test: + needs: discover_versions + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false name: SEED-SRC Test runs-on: ubuntu-22.04 timeout-minutes: 20 strategy: + fail-fast: false matrix: - wolfssl_ref: [ - 'master', - 'v5.8.4-stable'] - openssl_ref: [ - 'openssl-3.5.4', - 'openssl-3.0.17'] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: + - ${{ needs.discover_versions.outputs.openssl_latest_ref }} + - openssl-3.0.17 steps: - name: Checkout wolfProvider diff --git a/.github/workflows/simple.yml b/.github/workflows/simple.yml index 19c30ab4..24ae34f6 100644 --- a/.github/workflows/simple.yml +++ b/.github/workflows/simple.yml @@ -6,6 +6,18 @@ on: branches: [ 'master', 'main', 'release/**' ] pull_request: branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -13,24 +25,27 @@ concurrency: # END OF COMMON SECTION jobs: + discover_versions: + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false + uses: ./.github/workflows/_discover-versions.yml + simple_test: + needs: discover_versions + if: github.event_name != 'pull_request' || github.event.pull_request.draft == false name: Simple Test runs-on: ubuntu-22.04 timeout-minutes: 20 strategy: + fail-fast: false matrix: - wolfssl_ref: [ - 'master', - 'v5.8.4-stable'] - # Test against the newest of each minor version - openssl_ref: [ - 'openssl-3.5.4', - 'openssl-3.4.2', - 'openssl-3.3.4', - 'openssl-3.2.5', - 'openssl-3.1.8', - 'openssl-3.0.17'] - debug: ['', '--debug'] + # 2 wolfssl (master + latest-stable, resolved at run time) x + # 2 openssl (latest upstream release + oldest 3.0.x LTS) + # x 2 replace-default = 8 jobs. + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: + - ${{ needs.discover_versions.outputs.openssl_latest_ref }} + - openssl-3.0.17 + debug: [''] replace_default: [ '', '--replace-default --enable-replace-default-testing'] @@ -43,7 +58,9 @@ jobs: - name: Build and test wolfProvider run: | - OPENSSL_TAG=${{ matrix.openssl_ref }} WOLFSSL_TAG=${{ matrix.wolfssl_ref }} ./scripts/build-wolfprovider.sh ${{ matrix.debug }} ${{ matrix.replace_default }} + OPENSSL_TAG=${{ matrix.openssl_ref }} \ + WOLFSSL_TAG=${{ matrix.wolfssl_ref }} \ + ./scripts/build-wolfprovider.sh ${{ matrix.debug }} ${{ matrix.replace_default }} - name: Print errors if: ${{ failure() }} diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml new file mode 100644 index 00000000..ea1a2b60 --- /dev/null +++ b/.github/workflows/smoke-test.yml @@ -0,0 +1,72 @@ +name: Smoke Test + +# Fast pre-flight build + test for wolfProvider against a single +# wolfSSL/OpenSSL combo. Intentionally runs on drafts too: this is the +# gate that protects the rest of CI from broken commits. Other PR +# workflows can `uses:` the companion .github/actions/wait-for-smoke +# action to require this to pass before their expensive build matrices +# kick off. + +on: + push: + branches: [ 'master', 'main', 'release/**' ] + pull_request: + branches: [ '*' ] + types: [opened, synchronize, reopened, ready_for_review] + paths-ignore: + - '**.md' + - 'docs/**' + - 'LICENSE*' + - '.github/ISSUE_TEMPLATE/**' + - '.github/dependabot.yml' + - '.gitignore' + - 'AUTHORS' + - 'COPYING' + - 'README*' + - 'CHANGELOG*' + +concurrency: + group: smoke-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + + smoke: + needs: discover_versions + name: Smoke build (${{ matrix.name }}) + runs-on: ubuntu-22.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - name: master/openssl-latest + wolfssl_ref: master + - name: stable/openssl-latest + wolfssl_ref: '' # filled in from needs.discover_versions + + steps: + - name: Checkout wolfProvider + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Build and test wolfProvider + run: | + # Substitute the resolved latest-stable for the "stable" matrix row. + WOLFSSL_TAG="${{ matrix.wolfssl_ref || needs.discover_versions.outputs.wolfssl_ref }}" + OPENSSL_TAG=${{ needs.discover_versions.outputs.openssl_latest_ref }} \ + WOLFSSL_TAG="$WOLFSSL_TAG" \ + ./scripts/build-wolfprovider.sh + + - name: Print errors + if: ${{ failure() }} + run: | + if [ -f test-suite.log ] ; then + cat test-suite.log + fi diff --git a/.github/workflows/socat.yml b/.github/workflows/socat.yml index 1abeadfe..062eecd7 100644 --- a/.github/workflows/socat.yml +++ b/.github/workflows/socat.yml @@ -1,19 +1,20 @@ name: Socat Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Socat Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,18 +22,19 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_socat: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] continue-on-error: true container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. @@ -41,11 +43,10 @@ jobs: fail-fast: false matrix: socat_ref: [ 'socat-1.8.0.0' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -86,14 +87,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y git sudo build-essential autoconf automake \ - libtool pkg-config libjansson-dev check ca-certificates dpkg-dev \ - clang libc++-dev curl net-tools netcat-openbsd procps - - name: Download socat run: curl -O http://www.dest-unreach.org/socat/download/${{ matrix.socat_ref }}.tar.gz && tar xvf ${{ matrix.socat_ref }}.tar.gz @@ -132,10 +125,16 @@ jobs: # Show socat version (includes OpenSSL version info) ./socat -V - export ${{ matrix.force_fail }} - set +e - # Run the tests with expected failures - SOCAT=$GITHUB_WORKSPACE/${{ matrix.socat_ref }}/socat ./test.sh -t 0.5 --expect-fail 36,64,146,214,216,217,309,310,386,399,402,403,408,409,410,416,417,418,451,452,453,459,460,467,468,475,476,477,478,491,492,526,527,528,529,530 + EXPECTED_FAILS=36,64,146,214,216,217,309,310,386,399,402,403,408,409,410,416,417,418,451,452,453,459,460,467,468,475,476,477,478,491,492,526,527,528,529,530 + + # --- normal mode --- + SOCAT=$GITHUB_WORKSPACE/${{ matrix.socat_ref }}/socat ./test.sh -t 0.5 --expect-fail $EXPECTED_FAILS + TEST_RESULT=$? + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" socat + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + SOCAT=$GITHUB_WORKSPACE/${{ matrix.socat_ref }}/socat ./test.sh -t 0.5 --expect-fail $EXPECTED_FAILS TEST_RESULT=$? - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} socat + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" socat diff --git a/.github/workflows/sscep.yml b/.github/workflows/sscep.yml index 4ea28a9b..1640afd3 100644 --- a/.github/workflows/sscep.yml +++ b/.github/workflows/sscep.yml @@ -1,19 +1,20 @@ name: sscep Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for sscep Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,18 +22,19 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_sscep: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] timeout-minutes: 10 container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm options: --user root env: DEBIAN_FRONTEND: noninteractive @@ -40,13 +42,14 @@ jobs: OPENSSL_PACKAGES_PATH: /tmp/openssl-packages WOLFPROV_PACKAGES_PATH: /tmp/wolfprov-packages strategy: + fail-fast: false matrix: sscep_ref: [ 'v0.10.0' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] + # force_fail collapsed into sequential runs in the test step steps: - name: Checkout wolfProvider @@ -84,11 +87,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install sscep dependencies - run: | - apt-get update - apt-get install -y scep psmisc build-essential autoconf libtool pkg-config - - name: Download sscep uses: actions/checkout@v4 with: @@ -109,7 +107,13 @@ jobs: - name: Run sscep tests run: | - export ${{ matrix.force_fail }} - export WOLFPROV_FORCE_FAIL_STR="${{ matrix.force_fail }}" + cd sscep + + # --- normal mode --- + WOLFPROV_FORCE_FAIL_STR="" \ + $GITHUB_WORKSPACE/.github/scripts/test_sscep.sh - cd sscep && $GITHUB_WORKSPACE/.github/scripts/test_sscep.sh + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + WOLFPROV_FORCE_FAIL_STR="WOLFPROV_FORCE_FAIL=1" \ + $GITHUB_WORKSPACE/.github/scripts/test_sscep.sh diff --git a/.github/workflows/sssd.yml b/.github/workflows/sssd.yml index ceb6d80f..506ff782 100644 --- a/.github/workflows/sssd.yml +++ b/.github/workflows/sssd.yml @@ -1,16 +1,13 @@ name: SSSD Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for SSSD Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: test_sssd: @@ -26,10 +23,10 @@ jobs: sssd_ref: [ '2.9.1' ] wolfssl_ref: [ 'master', 'v5.8.0-stable' ] openssl_ref: [ 'openssl-3.5.0' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] - exclude: - - sssd_ref: 'master' - force_fail: 'WOLFPROV_FORCE_FAIL=1' + # NOTE: previously this matrix excluded sssd_ref=master + force_fail=1. + # force_fail is no longer a matrix axis (collapsed into sequential + # steps below). The skip is now expressed inline in the test step + # via `if [ "$SSSD_REF" = "master" ]`. steps: - name: Checkout wolfProvider uses: actions/checkout@v4 @@ -83,22 +80,37 @@ jobs: working-directory: sssd shell: bash run: | - # Set environment variables + # Set environment variables (shared across both modes) export LD_LIBRARY_PATH=$GITHUB_WORKSPACE/wolfssl-install/lib:$GITHUB_WORKSPACE/openssl-install/lib64 export OPENSSL_CONF=$GITHUB_WORKSPACE/provider.conf export OPENSSL_MODULES=$GITHUB_WORKSPACE/wolfprov-install/lib - export ${{ matrix.force_fail }} echo "Checking OpenSSL providers:" $GITHUB_WORKSPACE/openssl-install/bin/openssl list -providers | tee provider-list.log grep -q libwolfprov provider-list.log || (echo "ERROR: libwolfprov not found in OpenSSL providers" && exit 1) - # If force fail is enabled dont exit with error - if [ "${{ matrix.force_fail }}" == "WOLFPROV_FORCE_FAIL=1" ]; then - set +e - fi + # --- normal mode --- + make check 2>&1 | tee sssd-test.log + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" sssd - # Run tests and save result + # --- force-fail mode --- + # The original matrix had: + # exclude: + # - sssd_ref: 'master' + # force_fail: 'WOLFPROV_FORCE_FAIL=1' + # but sssd_ref never had a 'master' value, so that exclude + # was dead code. The intended skip is wolfssl_ref=master x + # force_fail (sssd hangs/breaks under WPFF on wolfssl master). + # Skip the force-fail round when wolfssl_ref=master to match + # that intent. + if [ "${{ matrix.wolfssl_ref }}" = "master" ]; then + echo "Skipping force-fail round for wolfssl_ref=master" + exit 0 + fi + export WOLFPROV_FORCE_FAIL=1 + set +e make check 2>&1 | tee sssd-test.log TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} sssd + set -e + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" sssd diff --git a/.github/workflows/stunnel.yml b/.github/workflows/stunnel.yml index cae41223..b50d520d 100644 --- a/.github/workflows/stunnel.yml +++ b/.github/workflows/stunnel.yml @@ -1,19 +1,20 @@ name: Stunnel Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for Stunnel Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_stunnel: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 10 strategy: + fail-fast: false matrix: stunnel_ref: [ 'stunnel-5.67' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,13 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y build-essential autoconf automake \ - autoconf-archive libtool libwrap0-dev pkg-config python3-venv \ - python3-cryptography patch git - - name: Check Python version run: python3 --version @@ -135,7 +130,7 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- # enter venv source myenv/bin/activate @@ -154,8 +149,43 @@ jobs: # Use `timeout` since the tests hang with WOLFPROV_FORCE_FAIL=1 timeout 10 make check 2>&1 || true - # grep for "failed: 0" in the results log, indicating success - TEST_RESULT=$(grep -c "failed: 0" tests/logs/results.log || echo 1) + # Mirror the results log so check-workflow-result.sh can find it. + cp -f tests/logs/results.log stunnel-test.log 2>/dev/null || true + + # "failed: 0" present in results.log == success + if grep -q "failed: 0" tests/logs/results.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + echo "Test result: $TEST_RESULT" + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" stunnel + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + + # enter venv + source myenv/bin/activate + + # Set this variable to prevent attempts to load the legacy OpenSSL + # provider, which we don't support. + # This is necessary for OpenSSL 3.0+ to avoid errors related to legacy + # algorithms that are not supported by wolfProvider. + export CRYPTOGRAPHY_OPENSSL_NO_LEGACY=1 + + # Verify stunnel + ./src/stunnel -version + + # Run tests + # Results captured in tests/logs/results.log + # Use `timeout` since the tests hang with WOLFPROV_FORCE_FAIL=1 + timeout 10 make check 2>&1 || true + cp -f tests/logs/results.log stunnel-test.log 2>/dev/null || true + + if grep -q "failed: 0" tests/logs/results.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi echo "Test result: $TEST_RESULT" - - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} stunnel + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" stunnel diff --git a/.github/workflows/systemd.yml b/.github/workflows/systemd.yml index 4ae223fd..e6c622d2 100644 --- a/.github/workflows/systemd.yml +++ b/.github/workflows/systemd.yml @@ -1,19 +1,20 @@ name: systemd Tests -# START OF COMMON SECTION -on: - push: - branches: ['master', 'main', 'release/**'] - pull_request: - branches: ['*'] +# OSP integration test for systemd Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,19 +22,20 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_systemd: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # This should be a safe limit for the tests to run. timeout-minutes: 20 container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -43,10 +45,9 @@ jobs: fail-fast: false matrix: systemd_ref: [ 'v254' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] steps: - name: Checkout wolfProvider @@ -84,19 +85,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - export DEBIAN_FRONTEND=noninteractive - apt-get update - apt-get install -y build-essential meson ninja-build \ - libmount-dev gperf python3-pytest python3-jinja2 python3-pip \ - libuv1-dev libnghttp2-dev libcap-dev uuid-dev libdevmapper-dev \ - libpopt-dev libjson-c-dev libargon2-dev libblkid-dev asciidoctor \ - pkgconf zlib1g-dev libgcrypt20-dev libgpg-error-dev libgnutls28-dev \ - libp11-kit-dev libfido2-dev libtss2-dev libdw-dev libbz2-dev \ - liblzma-dev liblz4-dev libzstd-dev libxkbcommon-dev libglib2.0-dev \ - libdbus-1-dev python3-setuptools python3-wheel git - - name: Checkout systemd uses: actions/checkout@v4 with: @@ -122,11 +110,21 @@ jobs: test-dns-packet test-dnssec test-resolve-tables \ test-resolved-etc-hosts test-resolved-packet \ test-resolved-stream" - export ${{ matrix.force_fail }} + # --- normal mode --- + meson test -C build $TEST_CASES + TEST_RESULT=$? + if [ $TEST_RESULT -ne 0 ]; then + cat build/meson-logs/testlog.txt + fi + + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" systemd + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 meson test -C build $TEST_CASES TEST_RESULT=$? if [ $TEST_RESULT -ne 0 ]; then cat build/meson-logs/testlog.txt fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} systemd + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" systemd diff --git a/.github/workflows/tcpdump.yml b/.github/workflows/tcpdump.yml index 295a4b09..a116f976 100644 --- a/.github/workflows/tcpdump.yml +++ b/.github/workflows/tcpdump.yml @@ -1,16 +1,20 @@ name: tcpdump Tests +# OSP integration test for tcpdump Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. + on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -18,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_tcpdump: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] continue-on-error: true container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 15 strategy: + fail-fast: false matrix: tcpdump_ref: [ 'tcpdump-4.99.3' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -81,12 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install test dependencies - run: | - apt-get update - apt-get install -y build-essential flex bison autoconf libtool\ - libpcap-dev - - name: Checkout tcpdump uses: actions/checkout@v4 with: @@ -128,10 +127,19 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run tests + make check 2>&1 | tee tcpdump-test.log + # Capture the test result using PIPESTATUS (Bash only) + TEST_RESULT=${PIPESTATUS[0]} + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" tcpdump + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run tests make check 2>&1 | tee tcpdump-test.log # Capture the test result using PIPESTATUS (Bash only) TEST_RESULT=${PIPESTATUS[0]} - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} tcpdump + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" tcpdump diff --git a/.github/workflows/tnftp.yml b/.github/workflows/tnftp.yml index 6beaf3e8..7cb3a77a 100644 --- a/.github/workflows/tnftp.yml +++ b/.github/workflows/tnftp.yml @@ -1,19 +1,20 @@ name: tnftp Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for tnftp Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,28 +22,29 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_tnftp: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: tnftp_ref: [ 'tnftp-20210827' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: ['WOLFPROV_FORCE_FAIL=1', ''] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,12 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install dependencies - run: | - apt-get update - apt-get install -y build-essential autoconf libtool pkg-config \ - vsftpd wget libncurses5-dev libncursesw5-dev - - name: Download and extract tnftp run: | # Fetch from the Debian source archive rather than ftp.netbsd.org @@ -110,7 +106,43 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Configure with OpenSSL + ./configure + + # Build tnftp + make -j + + # Run all tests and capture output + { + echo "Testing tnftp basic functionality..." + + # Test help command + if ./src/tnftp -? 2>&1 | grep -q "usage:"; then + echo "tnftp help command works" + else + echo "tnftp help command failed" + exit 1 + fi + + # Test that tnftp can start (even if it fails to connect) + echo "Testing tnftp connection attempt..." + timeout 10 ./src/tnftp -n 192.0.2.1 2>&1 | head -10 + echo "tnftp can attempt connections" + + # Test SSL/TLS functionality + echo "Testing SSL/TLS connection..." + timeout 15 ./src/tnftp -n https://httpbin.org/get 2>&1 + echo "SSL/TLS test completed" + } 2>&1 | tee tnftp-test.log + + # Capture result and check for expected failure + TEST_RESULT=$(grep -q "SSL context creation failed" tnftp-test.log && echo "1" || echo "0") + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" tnftp + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Configure with OpenSSL ./configure @@ -143,4 +175,4 @@ jobs: # Capture result and check for expected failure TEST_RESULT=$(grep -q "SSL context creation failed" tnftp-test.log && echo "1" || echo "0") - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} tnftp + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" tnftp diff --git a/.github/workflows/tpm2-tools.yml b/.github/workflows/tpm2-tools.yml index 76e71b00..8503c0ee 100644 --- a/.github/workflows/tpm2-tools.yml +++ b/.github/workflows/tpm2-tools.yml @@ -1,19 +1,20 @@ name: tpm2-tools Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for tpm2-tools Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,17 +22,18 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_tpm2_tools: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. @@ -40,10 +42,9 @@ jobs: fail-fast: false matrix: tpm2_tools_ref: [ '5.7' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -85,14 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install tpm2-tools test dependencies - run: | - apt-get update - apt-get install -y git build-essential expect vim dbus vim-common \ - autoconf-archive python3 python3-yaml python3-pip libefivar-dev \ - libcmocka-dev automake libtool pkg-config build-essential pandoc \ - libtss2-dev tpm2-abrmd swtpm tpm2-tools iproute2 libcurl4-openssl-dev - - name: Download tpm2-tools uses: actions/checkout@v4 with: @@ -115,7 +108,24 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} + # --- normal mode --- + + # Run only unit tests and integration tests that dont need TPM2 hardware/simulator + make check TESTS="test/unit/test_string_bytes test/unit/test_files \ + test/unit/test_tpm2_header test/unit/test_tpm2_attr_util test/unit/test_tpm2_alg_util \ + test/unit/test_pcr test/unit/test_tpm2_auth_util test/unit/test_tpm2_errata \ + test/unit/test_tpm2_session test/unit/test_tpm2_policy test/unit/test_tpm2_util \ + test/unit/test_options test/unit/test_cc_util test/unit/test_tpm2_eventlog \ + test/unit/test_tpm2_eventlog_yaml test/unit/test_object \ + test/integration/tests/X509certutil test/integration/tests/toggle_options \ + test/integration/tests/rc_decode test/integration/tests/X509certutil" 2>&1 | tee tpm2-tools-test.log + + # Capture result - Fails test/unit/test_tpm2_policy and test/unit/test_tpm2_eventlog with WPFF + TEST_RESULT=$(grep -q "# PASS: 20" tpm2-tools-test.log && echo "0" || echo "1") + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" tpm2-tools + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 # Run only unit tests and integration tests that dont need TPM2 hardware/simulator make check TESTS="test/unit/test_string_bytes test/unit/test_files \ @@ -129,4 +139,4 @@ jobs: # Capture result - Fails test/unit/test_tpm2_policy and test/unit/test_tpm2_eventlog with WPFF TEST_RESULT=$(grep -q "# PASS: 20" tpm2-tools-test.log && echo "0" || echo "1") - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} tpm2-tools + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" tpm2-tools diff --git a/.github/workflows/x11vnc.yml b/.github/workflows/x11vnc.yml index 40c3cb44..8a36242f 100644 --- a/.github/workflows/x11vnc.yml +++ b/.github/workflows/x11vnc.yml @@ -1,19 +1,20 @@ name: x11vnc Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - pull_request: - branches: [ '*' ] +# OSP integration test for x11vnc Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: + needs: discover_versions uses: ./.github/workflows/build-wolfprovider.yml with: wolfssl_ref: ${{ matrix.wolfssl_ref }} @@ -21,27 +22,28 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_x11vnc: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive timeout-minutes: 10 strategy: + fail-fast: false matrix: x11vnc_ref: [ '0.9.17' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages @@ -84,23 +86,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install x11vnc dependencies - run: | - apt-get update - - # common build dependencies - apt-get install -y build-essential autoconf automake libtool \ - pkg-config gcc make ca-certificates - - # x11vnc dependencies - apt-get install -y libc6-dev libjpeg-dev x11proto-core-dev \ - libxss-dev zlib1g-dev libavahi-client-dev libvncserver-dev \ - libx11-dev libxdamage-dev libxext-dev libxfixes-dev libxi-dev \ - libxinerama-dev libxrandr-dev libxtst-dev - - # packages for testing script - apt-get install -y xvfb tigervnc-viewer psmisc expect curl - - name: Download x11vnc uses: actions/checkout@v4 with: @@ -129,19 +114,36 @@ jobs: - name: Run x11vnc tests shell: bash run: | - export ${{ matrix.force_fail }} - export WOLFPROV_FORCE_FAIL_STR="${{ matrix.force_fail }}" - export X11VNC_TEST_LOG=/tmp/x11vnc-test.log export X11VNC_TEST_STATUS=0 - if ! $GITHUB_WORKSPACE/.github/scripts/x11vnc/test_x11vnc.sh $X11VNC_TEST_LOG; then + + run_round() { + local mode_label="$1" + local ff_arg="$2" + local log="/tmp/x11vnc-test-${mode_label}.log" + local status=0 + if ! $GITHUB_WORKSPACE/.github/scripts/x11vnc/test_x11vnc.sh "$log"; then + status=1 + fi + if $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh "$status" "$ff_arg" x11vnc; then + return 0 + fi + return 1 + } + + # --- normal mode --- + if ! run_round normal ""; then X11VNC_TEST_STATUS=1 fi - - if $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $X11VNC_TEST_STATUS "$WOLFPROV_FORCE_FAIL_STR" x11vnc; then - X11VNC_TEST_STATUS=0 - else + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + if ! run_round ff "WOLFPROV_FORCE_FAIL=1"; then X11VNC_TEST_STATUS=1 fi + unset WOLFPROV_FORCE_FAIL + + # Surface the most recent log via the existing follow-up step + export X11VNC_TEST_LOG=/tmp/x11vnc-test.log - name: Show x11vnc test log on failure run: | diff --git a/.github/workflows/xmlsec.yml b/.github/workflows/xmlsec.yml index decb647e..a26ff21d 100644 --- a/.github/workflows/xmlsec.yml +++ b/.github/workflows/xmlsec.yml @@ -1,18 +1,18 @@ name: xmlsec Tests -# START OF COMMON SECTION -on: - push: - branches: [ 'master', 'main', 'release/**' ] - #pull_request: - #branches: [ '*' ] +# OSP integration test for xmlsec Tests. Runs nightly via the +# Nightly OSP Suite orchestrator (.github/workflows/nightly-osp.yml) +# or manually via workflow_dispatch. NOT triggered on PR/push -- +# PR CI focuses on smoke + simple + cheap internal checks. -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true -# END OF COMMON SECTION +on: + workflow_call: {} + workflow_dispatch: {} jobs: + discover_versions: + uses: ./.github/workflows/_discover-versions.yml + build_wolfprovider: uses: ./.github/workflows/build-wolfprovider.yml with: @@ -21,30 +21,32 @@ jobs: fips_ref: ${{ matrix.fips_ref }} replace_default: ${{ matrix.replace_default }} strategy: + fail-fast: false matrix: - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] replace_default: [ true ] test_xmlsec: runs-on: ubuntu-22.04 - needs: build_wolfprovider + needs: [build_wolfprovider, discover_versions] # Run inside Debian Bookworm to match packaging environment container: - image: debian:bookworm + image: ghcr.io/wolfssl/wolfprovider-test-deps:bookworm env: DEBIAN_FRONTEND: noninteractive # This should be a safe limit for the tests to run. timeout-minutes: 20 strategy: + fail-fast: false matrix: xmlsec_ref: [ 'xmlsec-1_2_37' ] - wolfssl_ref: [ 'v5.8.4-stable' ] - openssl_ref: [ 'openssl-3.5.4' ] + wolfssl_ref: ${{ fromJson(needs.discover_versions.outputs.wolfssl_ref_array) }} + openssl_ref: ${{ fromJson(needs.discover_versions.outputs.openssl_ref_array) }} fips_ref: [ 'FIPS', 'non-FIPS' ] - force_fail: [ 'WOLFPROV_FORCE_FAIL=1', '' ] replace_default: [ true ] + # force_fail collapsed into sequential runs in the test step env: WOLFSSL_PACKAGES_PATH: /tmp/wolfssl-packages OPENSSL_PACKAGES_PATH: /tmp/openssl-packages @@ -85,13 +87,6 @@ jobs: ${{ matrix.replace_default && '--replace-default' || '' }} \ ${{ matrix.fips_ref == 'FIPS' && '--fips' || '' }} - - name: Install xmlsec dependencies - run: | - apt-get update - apt-get install -y automake autoconf libtool libtool-bin \ - libltdl-dev libltdl7 libxml2-dev patch build-essential \ - pkg-config libxml2-dev - - name: Download xmlsec uses: actions/checkout@v4 with: @@ -133,12 +128,24 @@ jobs: shell: bash run: | set +o pipefail # ignore errors from make check - export ${{ matrix.force_fail }} - make check-keys | tee xmlsec-keys.log - make check-enc | tee xmlsec-enc.log - if grep -q "TOTAL FAILED: 0" xmlsec-enc.log && grep -q "TOTAL FAILED: 0" xmlsec-keys.log; then + + # --- normal mode --- + make check-keys | tee xmlsec-keys-normal.log + make check-enc | tee xmlsec-enc-normal.log + if grep -q "TOTAL FAILED: 0" xmlsec-enc-normal.log && grep -q "TOTAL FAILED: 0" xmlsec-keys-normal.log; then + TEST_RESULT=0 + else + TEST_RESULT=1 + fi + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "" xmlsec + + # --- force-fail mode --- + export WOLFPROV_FORCE_FAIL=1 + make check-keys | tee xmlsec-keys-ff.log + make check-enc | tee xmlsec-enc-ff.log + if grep -q "TOTAL FAILED: 0" xmlsec-enc-ff.log && grep -q "TOTAL FAILED: 0" xmlsec-keys-ff.log; then TEST_RESULT=0 else TEST_RESULT=1 fi - $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT ${{ matrix.force_fail }} xmlsec + $GITHUB_WORKSPACE/.github/scripts/check-workflow-result.sh $TEST_RESULT "WOLFPROV_FORCE_FAIL=1" xmlsec diff --git a/docker/wolfprovider-test-deps/Dockerfile b/docker/wolfprovider-test-deps/Dockerfile new file mode 100644 index 00000000..9d4d14d2 --- /dev/null +++ b/docker/wolfprovider-test-deps/Dockerfile @@ -0,0 +1,97 @@ +# wolfProvider test-deps container. +# +# Bakes in the union of apt packages every PR workflow used to install at +# job time. Goal: zero `apt-get update` calls during PR CI. Built and +# pushed to ghcr.io/wolfssl/wolfprovider-test-deps:bookworm by the +# publish-test-deps-image.yml workflow. + +FROM debian:bookworm-slim + +ENV DEBIAN_FRONTEND=noninteractive +ENV PERL_MM_USE_DEFAULT=1 + +# One apt-get update + one apt-get install. Anything you add here ships +# in the image; do not add per-workflow installs. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + # core build toolchain + build-essential gcc g++ make m4 gettext \ + autoconf automake autoconf-archive autopoint autotools-dev \ + libtool libtool-bin pkg-config pkgconf \ + cmake meson ninja-build bison flex gperf gengetopt help2man \ + clang clang-tools cppcheck libc++-dev \ + # vcs, transport, compression, scripting + git git-all curl wget ca-certificates patch xxd dpkg-dev \ + gawk perl python3 python3-pip python3-venv python3-wheel \ + python3-setuptools python3-yaml python3-jinja2 python3-six \ + python3-pytest python3-dnspython python3-hypothesis \ + python3-impacket python3-cryptography python3-pycryptodome \ + python3-cmarkgfm python3-docutils python-dev-is-python3 \ + cpanminus \ + # editors, debug, system bits + valgrind vim vim-common groff sudo procps psmisc bc less \ + bsdextrautils util-linux kmod systemd cryptsetup-bin \ + ca-certificates dbus \ + # networking, ssh, vnc, sniffers + net-tools netcat-openbsd iproute2 iw bridge-utils \ + openssh-client openssh-server expect \ + tigervnc-viewer xvfb \ + wireless-regdb wireless-tools \ + # zlib / xz / lzma / bz2 / lzo / lz4 / zstd + zlib1g zlib1g-dev liblzma-dev libbz2-dev liblzo2-dev \ + liblz4-dev libzstd-dev libzstd1 \ + # crypto / TLS adjacents. libssl3/libssl-dev are the *stock* Debian + # versions; replace-default mode dpkg-installs wolfprov-patched + # debs on top (and apt-marks them held) at job time. + openssl libssl-dev libssl3 \ + libgcrypt20-dev libgpg-error-dev \ + libgnutls28-dev gnutls-bin \ + libp11-dev libp11-kit-dev libargon2-dev libcbor-dev \ + libcurl4-openssl-dev libidn2-dev libnghttp2-dev nghttp2 \ + libpsl-dev libpsl5 \ + libpcsclite-dev opensc pcsc-tools pcscd \ + libtss2-dev tpm2-tools tpm2-abrmd swtpm softhsm2 \ + libfido2-dev \ + libsasl2-dev libldap2-dev libldb-dev libldb2 \ + libpcre2-dev libpcre3-dev \ + libreadline-dev libsqlite3-dev libpopt-dev libpcap-dev libpcap0.8 \ + libseccomp-dev libwrap0-dev libudev-dev libdevmapper-dev libcap-dev \ + libcap-ng-dev libcap2 libacl1-dev libattr1-dev libblkid-dev \ + libmount-dev libdw-dev libdbus-1-dev libglib2.0-dev \ + libgstreamer1.0-dev gstreamer1.0-plugins-base-apps \ + libhiredis-dev libjansson-dev libjemalloc-dev libjson-c-dev \ + libxml2-dev libcunit1-dev libcmocka-dev libpam0g-dev \ + libpng-dev libjpeg-dev libusb-1.0-0-dev libuv1-dev libverto-dev \ + libavahi-client-dev libavahi-compat-libdnssd-dev libmemcached-dev \ + libutf8proc-dev libxxhash-dev libkeyutils-dev libcom-err2 \ + libcjose-dev libeac-dev libefivar-dev libncurses5-dev \ + libncursesw5-dev libiberty-dev libltdl-dev libltdl7 \ + libperl-dev linux-libc-dev binutils-dev uuid-dev \ + # X11 (x11vnc, qt5network5) + libx11-dev libxdamage-dev libxext-dev libxfixes-dev libxi-dev \ + libxinerama-dev libxrandr-dev libxss-dev libxtst-dev \ + libxkbcommon-dev libvncserver-dev x11proto-core-dev \ + # netlink (hostap) + libnl-3-dev libnl-route-3-dev libnl-genl-3-dev libnl-genl-3-200 \ + # krb5 build deps (e2fsprogs headers) + comerr-dev ss-dev libss2 \ + # sscep test + scep \ + # extras called out by specific workflows + check apache2-dev acl attr fakeroot \ + asciidoctor docbook-xsl pandoc xsltproc man2html \ + memcached vsftpd pps-tools tshark \ + && rm -rf /var/lib/apt/lists/* + +# Perl modules used by app test suites. Baked in so transient CPAN +# flakes don't poison a PR run. +RUN cpanm --notest \ + Net::DNS \ + Proc::Find Net::SSLeay IO::Socket::SSL \ + && rm -rf /root/.cpanm + +# OpenSSL/wolfSSL come from .deb packages pulled at job time via ORAS. +# We do NOT pre-install them here; the workflows do +# dpkg -i $WOLFSSL_PACKAGES_PATH/libwolfssl_*.deb +# and then apt-mark hold them to stop test-dep apt installs from +# clobbering the wolfprov-patched libssl3.