From 013e5d8302a48519023b340644c073e0b9890e01 Mon Sep 17 00:00:00 2001 From: Yicong Huang Date: Sat, 2 May 2026 10:47:30 -0700 Subject: [PATCH 1/6] refactor(ci): split scala job into amber + platform matrix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - build.yml: - Replace the `scala` job with `amber`. It runs the cross-cutting Scala lints (scalafmtCheckAll, scalafixAll --check) once on behalf of every Scala module, builds WorkflowExecutionService/dist, license-checks the amber dist, and runs amber tests via WorkflowExecutionService/jacoco. - Add `platform` job: a matrix over the five non-amber Scala services (config-service, access-control-service, file-service, computing-unit- managing-service, workflow-compiling-service). Each entry runs a single sbt invocation `/dist /test` and license- checks its own dist lib only. This is now possible because per-module LICENSE/NOTICE binaries (#4668) are embedded in each service's dist. - Replace the `run_scala` workflow_call input with `run_amber` and `run_platform`. - labeler.yml: introduce a `platform` label scoped to the five platform service dirs. Trim the `service` label to just `pyright-language-service` plus root-level Scala build/lint config (`build.sbt`, `project/**`, `.scalafix.conf`, `.scalafmt.conf`) — those affect amber + every platform service. - required-checks.yml: precheck now emits `run_amber` + `run_platform` outputs. LABEL_STACKS maps the new `platform` label to the platform stack and routes existing labels accordingly: python/engine -> amber + python; service/common/ddl-change -> amber + platform; ci -> all stacks. Build and backport callers pass the new inputs through. Closes #4631 --- .github/labeler.yml | 11 +- .github/workflows/build.yml | 160 ++++++++++++++++++-------- .github/workflows/required-checks.yml | 70 ++++++----- 3 files changed, 163 insertions(+), 78 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index defbdc90110..1ff9a16c35d 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -24,17 +24,22 @@ common: - any-glob-to-any-file: - 'common/**' -service: +platform: - changed-files: - any-glob-to-any-file: - 'access-control-service/**' - 'computing-unit-managing-service/**' - 'config-service/**' - 'file-service/**' - - 'pyright-language-service/**' - 'workflow-compiling-service/**' + +service: + - changed-files: + - any-glob-to-any-file: + - 'pyright-language-service/**' # Root-level scala build / lint config: a change to any of these - # affects the scala stack, but no component label catches them. + # affects every Scala stack (amber + the platform services), but + # no component label catches them. - 'build.sbt' - 'project/**' - '.scalafix.conf' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 45569e311c8..851cce7a63b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,7 +40,11 @@ on: required: false type: boolean default: true - run_scala: + run_amber: + required: false + type: boolean + default: true + run_platform: required: false type: boolean default: true @@ -121,9 +125,14 @@ jobs: flags: frontend fail_ci_if_error: false - scala: - if: ${{ inputs.run_scala }} - name: ${{ format('scala{0} ({1}, 11)', inputs.job_name_suffix, matrix.os) }} + amber: + # The amber job runs the cross-cutting Scala lints (scalafmtCheckAll, + # scalafixAll --check) once on behalf of every Scala module, then builds + # and tests just the WorkflowExecutionService dist. Per-service builds + # and tests for the platform services live in the `platform` matrix + # below. License-binary checks are scoped to the amber dist. + if: ${{ inputs.run_amber }} + name: ${{ format('amber{0} ({1}, 11)', inputs.job_name_suffix, matrix.os) }} strategy: matrix: os: [ubuntu-22.04] @@ -179,51 +188,28 @@ jobs: - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 # v8.1.0 with: extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' - - name: Lint and build distributable bundles - # Single sbt invocation runs scalafmt -> scalafix -> per-module dist - # in order; sbt exits at the first failing command (fail-fast). Each - # command is a separate sbt arg, not joined with ';', so a dist - # failure aborts the rest. scalafix triggers compile (and JOOQ - # codegen), which the dist commands then reuse incrementally. + - name: Lint and build amber distributable bundle + # Single sbt invocation: scalafmt -> scalafix -> amber dist. + # scalafmtCheckAll and scalafixAll cover every Scala module, so the + # platform matrix below skips them. scalafix triggers compile (and + # JOOQ codegen), which the dist command then reuses incrementally. run: | sbt scalafmtCheckAll \ "scalafixAll --check" \ - ConfigService/dist \ - AccessControlService/dist \ - FileService/dist \ - ComputingUnitManagingService/dist \ - WorkflowCompilingService/dist \ WorkflowExecutionService/dist - - name: Unzip dists and check binary licenses - # Unzips every service's dist bundle, runs the binding LICENSE-binary - # check, then runs the advisory per-dep audit. The audit always runs - # (mirroring the previous 'if: always()' on its own step) and never - # fails the step; the binding check's exit code drives the result. + - name: Unzip amber dist and check binary licenses + # Per-module LICENSE-binary lives inside the amber dist (#4668), so + # the binding check is scoped to amber's lib only. The audit always + # runs (mirroring the previous 'if: always()' on its own step) and + # never fails the step; the binding check's exit code drives it. run: | set -euo pipefail mkdir -p /tmp/dists - for zip in \ - config-service/target/universal/config-service-*.zip \ - access-control-service/target/universal/access-control-service-*.zip \ - file-service/target/universal/file-service-*.zip \ - computing-unit-managing-service/target/universal/computing-unit-managing-service-*.zip \ - workflow-compiling-service/target/universal/workflow-compiling-service-*.zip \ - amber/target/universal/amber-*.zip; do - unzip -q "$zip" -d /tmp/dists/ - done - - lib_paths=( - /tmp/dists/config-service-*/lib - /tmp/dists/access-control-service-*/lib - /tmp/dists/file-service-*/lib - /tmp/dists/computing-unit-managing-service-*/lib - /tmp/dists/workflow-compiling-service-*/lib - /tmp/dists/amber-*/lib - ) + unzip -q amber/target/universal/amber-*.zip -d /tmp/dists/ check_exit=0 - ./bin/licensing/check_binary_deps.py --ignore-transitive-version jar "${lib_paths[@]}" || check_exit=$? - ./bin/licensing/audit_jar_licenses.py "${lib_paths[@]}" || true + ./bin/licensing/check_binary_deps.py --ignore-transitive-version jar /tmp/dists/amber-*/lib || check_exit=$? + ./bin/licensing/audit_jar_licenses.py /tmp/dists/amber-*/lib || true exit "$check_exit" - name: Install dependencies # Only the backend test step needs the python deps; install just @@ -240,19 +226,101 @@ jobs: run: | echo "api.version=1.52" >> ~/.docker-java.properties cat ~/.docker-java.properties - - name: Run backend tests - # 'jacoco' runs tests under sbt-jacoco's JVM agent and emits per- - # module jacoco.xml that the codecov upload step picks up. - run: sbt jacoco - - name: Upload scala coverage to Codecov + - name: Run amber tests + # 'jacoco' runs tests under sbt-jacoco's JVM agent and emits + # jacoco.xml that the codecov upload step picks up. + run: sbt "WorkflowExecutionService/jacoco" + - name: Upload amber coverage to Codecov if: always() uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./**/target/scala-2.13/jacoco/report/jacoco.xml + files: ./amber/target/scala-2.13/jacoco/report/jacoco.xml flags: scala fail_ci_if_error: false + platform: + # Per-service build, test, and license check for the non-amber Scala + # services. Each matrix entry runs its own dist + test in isolation + # against per-module LICENSE-binary (#4668). scalafmt / scalafix already + # cover every module in the amber job above, so this matrix skips them. + if: ${{ inputs.run_platform }} + name: ${{ format('platform{0} ({1})', inputs.job_name_suffix, matrix.service) }} + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: + - service: config-service + sbt_project: ConfigService + - service: access-control-service + sbt_project: AccessControlService + - service: file-service + sbt_project: FileService + - service: computing-unit-managing-service + sbt_project: ComputingUnitManagingService + - service: workflow-compiling-service + sbt_project: WorkflowCompilingService + env: + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + services: + # Each platform service transitively depends on DAO, which runs JOOQ + # code generation at compile time and needs the live texera schema. + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + ref: ${{ inputs.checkout_ref || github.sha }} + fetch-depth: 0 + - name: Prepare backport workspace + if: ${{ inputs.backport_target_branch != '' }} + working-directory: ${{ github.workspace }} + run: bash ./.github/scripts/prepare-backport-checkout.sh "${{ inputs.backport_target_branch }}" "${{ inputs.backport_commit_range }}" + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: "temurin" + java-version: 11 + - name: Setup sbt launcher + uses: sbt/setup-sbt@508b753e53cb6095967669e0911487d2b9bc9f41 # v1.1.22 + - uses: coursier/cache-action@90c37294538be80a558fd665531fcdc2b467b475 # v8.1.0 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + - name: Create Databases + run: | + psql -h localhost -U postgres -f sql/texera_ddl.sql + psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql + psql -h localhost -U postgres -f sql/texera_lakefs.sql + env: + PGPASSWORD: postgres + - name: Build dist and run ${{ matrix.service }} tests + # Single sbt invocation so dist + test share compiled state. + run: sbt "${{ matrix.sbt_project }}/dist" "${{ matrix.sbt_project }}/test" + - name: Unzip ${{ matrix.service }} dist and check binary licenses + # Per-module LICENSE-binary lives inside this service's dist + # (#4668), so the check is scoped to its own lib. + run: | + set -euo pipefail + mkdir -p /tmp/dists + unzip -q ${{ matrix.service }}/target/universal/${{ matrix.service }}-*.zip -d /tmp/dists/ + + check_exit=0 + ./bin/licensing/check_binary_deps.py jar /tmp/dists/${{ matrix.service }}-*/lib || check_exit=$? + ./bin/licensing/audit_jar_licenses.py /tmp/dists/${{ matrix.service }}-*/lib || true + exit "$check_exit" + python: if: ${{ inputs.run_python }} name: ${{ format('python{0} ({1}, {2})', inputs.job_name_suffix, matrix.os, matrix.python-version) }} diff --git a/.github/workflows/required-checks.yml b/.github/workflows/required-checks.yml index 1429521985a..d20e2d6a1f2 100644 --- a/.github/workflows/required-checks.yml +++ b/.github/workflows/required-checks.yml @@ -46,11 +46,12 @@ jobs: # - On PR events, wait for the Pull Request Labeler workflow to finish so # the labels it applies (frontend, docs, dev, ...) are available, then # gate run_* outputs on those labels. - # - run_frontend / run_scala / run_python / run_agent_service: gate the - # main build stacks. Each labeler-applied label maps to the stacks it - # requires (LABEL_STACKS below); the run set is the union across all - # PR labels. Empty union (e.g. docs-only / dev-only PRs) skips every - # stack. Push and workflow_dispatch events run every stack. + # - run_frontend / run_amber / run_platform / run_python / + # run_agent_service: gate the main build stacks. Each labeler-applied + # label maps to the stacks it requires (LABEL_STACKS below); the run + # set is the union across all PR labels. Empty union (e.g. docs-only + # / dev-only PRs) skips every stack. Push and workflow_dispatch + # events run every stack. # - backport_targets: JSON array of release/* labels currently on the PR. # Drives the backport matrix; empty array means no backport runs. precheck: @@ -58,7 +59,8 @@ jobs: runs-on: ubuntu-latest outputs: run_frontend: ${{ steps.decide.outputs.run_frontend }} - run_scala: ${{ steps.decide.outputs.run_scala }} + run_amber: ${{ steps.decide.outputs.run_amber }} + run_platform: ${{ steps.decide.outputs.run_platform }} run_python: ${{ steps.decide.outputs.run_python }} run_agent_service: ${{ steps.decide.outputs.run_agent_service }} backport_targets: ${{ steps.decide.outputs.backport_targets }} @@ -113,31 +115,37 @@ jobs: // labeler matches lives under a component dir and is already // covered by that component's label. // - // label | frontend | scala | python | agent-service - // ---------------------|----------|-------|--------|-------------- - // frontend | x | | | - // python | | x | x | - // engine | | x | x | - // service | | x | | - // agent-service | | | | x - // common | | x | | - // ddl-change | | x | | - // ci | x | x | x | x - // docs / dev / deps / | | | | - // release/* / branch | | | | + // label | frontend | amber | platform | python | agent-service + // ---------------|----------|-------|----------|--------|-------------- + // frontend | x | | | | + // python | | x | | x | + // engine | | x | | x | + // platform | | | x | | + // service | | x | x | | (root scala + // build/lint + // config) + // agent-service | | | | | x + // common | | x | x | | + // ddl-change | | x | x | | + // ci | x | x | x | x | x + // docs / dev / | | | | | + // deps / release/| | | | | + // * / branch | | | | | const LABEL_STACKS = { frontend: ["frontend"], - python: ["scala", "python"], // pyamber drives scala integration tests too - engine: ["scala", "python"], // amber/** spans both - service: ["scala"], // scala-side services; agent-service is its own label + python: ["amber", "python"], // pyamber drives amber integration tests too + engine: ["amber", "python"], // amber/** spans both + platform: ["platform"], // platform services + service: ["amber", "platform"], // root scala build/lint affects both "agent-service": ["agent-service"], - common: ["scala"], - "ddl-change": ["scala"], - ci: ["frontend", "scala", "python", "agent-service"], + common: ["amber", "platform"], + "ddl-change": ["amber", "platform"], + ci: ["frontend", "amber", "platform", "python", "agent-service"], }; let runFrontend = true; - let runScala = true; + let runAmber = true; + let runPlatform = true; let runPython = true; let runAgentService = true; @@ -149,7 +157,8 @@ jobs: } } runFrontend = stacks.has("frontend"); - runScala = stacks.has("scala"); + runAmber = stacks.has("amber"); + runPlatform = stacks.has("platform"); runPython = stacks.has("python"); runAgentService = stacks.has("agent-service"); core.info( @@ -158,7 +167,8 @@ jobs: } core.setOutput("run_frontend", runFrontend ? "true" : "false"); - core.setOutput("run_scala", runScala ? "true" : "false"); + core.setOutput("run_amber", runAmber ? "true" : "false"); + core.setOutput("run_platform", runPlatform ? "true" : "false"); core.setOutput("run_python", runPython ? "true" : "false"); core.setOutput("run_agent_service", runAgentService ? "true" : "false"); @@ -211,7 +221,8 @@ jobs: uses: ./.github/workflows/build.yml with: run_frontend: ${{ needs.precheck.outputs.run_frontend == 'true' }} - run_scala: ${{ needs.precheck.outputs.run_scala == 'true' }} + run_amber: ${{ needs.precheck.outputs.run_amber == 'true' }} + run_platform: ${{ needs.precheck.outputs.run_platform == 'true' }} run_python: ${{ needs.precheck.outputs.run_python == 'true' }} run_agent_service: ${{ needs.precheck.outputs.run_agent_service == 'true' }} secrets: inherit @@ -230,7 +241,8 @@ jobs: backport_commit_range: ${{ format('{0}..{1}', github.event.pull_request.base.sha, github.event.pull_request.head.sha) }} job_name_suffix: "" run_frontend: ${{ needs.precheck.outputs.run_frontend == 'true' }} - run_scala: ${{ needs.precheck.outputs.run_scala == 'true' }} + run_amber: ${{ needs.precheck.outputs.run_amber == 'true' }} + run_platform: ${{ needs.precheck.outputs.run_platform == 'true' }} run_python: ${{ needs.precheck.outputs.run_python == 'true' }} run_agent_service: ${{ needs.precheck.outputs.run_agent_service == 'true' }} secrets: inherit From a37ecd14161850ee56260a6e8fc3d7941ad42d75 Mon Sep 17 00:00:00 2001 From: Yicong Huang Date: Sat, 2 May 2026 10:57:25 -0700 Subject: [PATCH 2/6] fix(ci): point per-service license check at per-module LICENSE-binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each service's dist still embeds the root union LICENSE-binary via AddMetaInfLicenseFiles, so a per-service `check_binary_deps.py jar /lib` resolved against the embedded file fails with STALE entries — the union lists jars not in any single service. Pass the per-module file (e.g. amber/LICENSE-binary-java, config-service/LICENSE-binary) introduced in #4668 via the script's `--license-binary` flag so each service is validated against its own ground-truth list. --- .github/workflows/build.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 851cce7a63b..ff6f9f941f7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -198,17 +198,20 @@ jobs: "scalafixAll --check" \ WorkflowExecutionService/dist - name: Unzip amber dist and check binary licenses - # Per-module LICENSE-binary lives inside the amber dist (#4668), so - # the binding check is scoped to amber's lib only. The audit always - # runs (mirroring the previous 'if: always()' on its own step) and - # never fails the step; the binding check's exit code drives it. + # Per-module LICENSE-binary files live at the repo root after #4668; + # the amber JVM dist is checked against amber/LICENSE-binary-java. + # The audit always runs (mirroring the previous 'if: always()' on its + # own step) and never fails the step; the binding check's exit code + # drives it. run: | set -euo pipefail mkdir -p /tmp/dists unzip -q amber/target/universal/amber-*.zip -d /tmp/dists/ check_exit=0 - ./bin/licensing/check_binary_deps.py --ignore-transitive-version jar /tmp/dists/amber-*/lib || check_exit=$? + ./bin/licensing/check_binary_deps.py --ignore-transitive-version jar \ + --license-binary amber/LICENSE-binary-java \ + /tmp/dists/amber-*/lib || check_exit=$? ./bin/licensing/audit_jar_licenses.py /tmp/dists/amber-*/lib || true exit "$check_exit" - name: Install dependencies @@ -309,15 +312,17 @@ jobs: # Single sbt invocation so dist + test share compiled state. run: sbt "${{ matrix.sbt_project }}/dist" "${{ matrix.sbt_project }}/test" - name: Unzip ${{ matrix.service }} dist and check binary licenses - # Per-module LICENSE-binary lives inside this service's dist - # (#4668), so the check is scoped to its own lib. + # Each platform service has its own LICENSE-binary at the repo root + # after #4668; check this service's dist against just its own file. run: | set -euo pipefail mkdir -p /tmp/dists unzip -q ${{ matrix.service }}/target/universal/${{ matrix.service }}-*.zip -d /tmp/dists/ check_exit=0 - ./bin/licensing/check_binary_deps.py jar /tmp/dists/${{ matrix.service }}-*/lib || check_exit=$? + ./bin/licensing/check_binary_deps.py jar \ + --license-binary ${{ matrix.service }}/LICENSE-binary \ + /tmp/dists/${{ matrix.service }}-*/lib || check_exit=$? ./bin/licensing/audit_jar_licenses.py /tmp/dists/${{ matrix.service }}-*/lib || true exit "$check_exit" From d8e08bf71ef1bc84c583e31539e59ff49684b6ab Mon Sep 17 00:00:00 2001 From: Yicong Huang Date: Sat, 2 May 2026 11:33:16 -0700 Subject: [PATCH 3/6] chore(ci): drop service label, fold root scala build/lint into common MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `service` label was carrying two unrelated things: pyright-language- service/** (TypeScript, no test stack) and the root-level Scala build/ lint config. Drop the label entirely: - pyright-language-service/** is now uncategorized — fine, nothing tests it today. - build.sbt / project/** / .scalafix.conf / .scalafmt.conf join the `common` glob; common already maps to amber + platform, which is correct for changes that affect every Scala module. --- .github/labeler.yml | 18 ++++++------------ .github/workflows/required-checks.yml | 9 ++++----- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index 1ff9a16c35d..1cfd4eef74f 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -23,6 +23,12 @@ common: - changed-files: - any-glob-to-any-file: - 'common/**' + # Root-level Scala build / lint config: a change to any of these + # affects every Scala stack (amber + the platform services). + - 'build.sbt' + - 'project/**' + - '.scalafix.conf' + - '.scalafmt.conf' platform: - changed-files: @@ -33,18 +39,6 @@ platform: - 'file-service/**' - 'workflow-compiling-service/**' -service: - - changed-files: - - any-glob-to-any-file: - - 'pyright-language-service/**' - # Root-level scala build / lint config: a change to any of these - # affects every Scala stack (amber + the platform services), but - # no component label catches them. - - 'build.sbt' - - 'project/**' - - '.scalafix.conf' - - '.scalafmt.conf' - agent-service: - changed-files: - any-glob-to-any-file: diff --git a/.github/workflows/required-checks.yml b/.github/workflows/required-checks.yml index d20e2d6a1f2..9046682bd73 100644 --- a/.github/workflows/required-checks.yml +++ b/.github/workflows/required-checks.yml @@ -121,11 +121,11 @@ jobs: // python | | x | | x | // engine | | x | | x | // platform | | | x | | - // service | | x | x | | (root scala + // agent-service | | | | | x + // common | | x | x | | (also catches + // root scala // build/lint // config) - // agent-service | | | | | x - // common | | x | x | | // ddl-change | | x | x | | // ci | x | x | x | x | x // docs / dev / | | | | | @@ -136,9 +136,8 @@ jobs: python: ["amber", "python"], // pyamber drives amber integration tests too engine: ["amber", "python"], // amber/** spans both platform: ["platform"], // platform services - service: ["amber", "platform"], // root scala build/lint affects both "agent-service": ["agent-service"], - common: ["amber", "platform"], + common: ["amber", "platform"], // common/** + root scala build/lint "ddl-change": ["amber", "platform"], ci: ["frontend", "amber", "platform", "python", "agent-service"], }; From 68ec385e5a34f72932d7ba285aac68764c6207ad Mon Sep 17 00:00:00 2001 From: Yicong Huang Date: Sat, 2 May 2026 11:46:21 -0700 Subject: [PATCH 4/6] fix(ci): restore common-module tests and platform coverage uploads The split lost two things vs. the previous single `sbt jacoco` run: - Common module tests didn't run anywhere. `WorkflowExecutionService/ jacoco` only runs that one project; sbt's `test` task doesn't transit dependsOn. Add explicit `DAO/jacoco`, `PyBuilder/jacoco`, `WorkflowCore/jacoco`, `WorkflowOperator/jacoco` to the amber job alongside `WorkflowExecutionService/jacoco` (Auth and Config have no tests so they're skipped). 64 test files (46 in workflow-operator, 15 in workflow-core, 2 in pybuilder, 1 in dao) are back. - Platform jobs ran `/test`, which produces no jacoco report, so coverage uploads silently dropped for the five platform services. Switch to `/jacoco` and add a per-service codecov upload step. Same `scala` flag as the amber job so Codecov merges all Scala reports into one view. Amber job's codecov upload step also broadens its file pattern from `./amber/.../jacoco.xml` to `./**/target/scala-2.13/jacoco/report/jacoco.xml` so the per-common-module reports are picked up too. --- .github/workflows/build.yml | 38 ++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ff6f9f941f7..f4fee9f9516 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -229,16 +229,25 @@ jobs: run: | echo "api.version=1.52" >> ~/.docker-java.properties cat ~/.docker-java.properties - - name: Run amber tests - # 'jacoco' runs tests under sbt-jacoco's JVM agent and emits - # jacoco.xml that the codecov upload step picks up. - run: sbt "WorkflowExecutionService/jacoco" - - name: Upload amber coverage to Codecov + - name: Run amber and common module tests with coverage + # 'jacoco' runs tests under sbt-jacoco's JVM agent and emits per- + # module jacoco.xml that the codecov upload step picks up. + # `WorkflowExecutionService/jacoco` only runs that project's tests + # (sbt's `test` task does not transit dependsOn), so common + # modules' tests are listed explicitly here. Modules with no + # tests (Auth, Config) are skipped. + run: | + sbt "DAO/jacoco" \ + "PyBuilder/jacoco" \ + "WorkflowCore/jacoco" \ + "WorkflowOperator/jacoco" \ + "WorkflowExecutionService/jacoco" + - name: Upload amber and common coverage to Codecov if: always() uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 with: token: ${{ secrets.CODECOV_TOKEN }} - files: ./amber/target/scala-2.13/jacoco/report/jacoco.xml + files: ./**/target/scala-2.13/jacoco/report/jacoco.xml flags: scala fail_ci_if_error: false @@ -308,9 +317,10 @@ jobs: psql -h localhost -U postgres -f sql/texera_lakefs.sql env: PGPASSWORD: postgres - - name: Build dist and run ${{ matrix.service }} tests - # Single sbt invocation so dist + test share compiled state. - run: sbt "${{ matrix.sbt_project }}/dist" "${{ matrix.sbt_project }}/test" + - name: Build dist and run ${{ matrix.service }} tests with coverage + # Single sbt invocation so dist + test share compiled state. Use + # `jacoco` so the codecov upload step has a report to pick up. + run: sbt "${{ matrix.sbt_project }}/dist" "${{ matrix.sbt_project }}/jacoco" - name: Unzip ${{ matrix.service }} dist and check binary licenses # Each platform service has its own LICENSE-binary at the repo root # after #4668; check this service's dist against just its own file. @@ -325,6 +335,16 @@ jobs: /tmp/dists/${{ matrix.service }}-*/lib || check_exit=$? ./bin/licensing/audit_jar_licenses.py /tmp/dists/${{ matrix.service }}-*/lib || true exit "$check_exit" + - name: Upload ${{ matrix.service }} coverage to Codecov + # Same `scala` flag as the amber job so Codecov merges all Scala + # coverage reports into a single view. + if: always() + uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./${{ matrix.service }}/target/scala-2.13/jacoco/report/jacoco.xml + flags: scala + fail_ci_if_error: false python: if: ${{ inputs.run_python }} From c6c031d8bf8b81d30fa6391f4eda358d5e292d32 Mon Sep 17 00:00:00 2001 From: Yicong Huang Date: Sat, 2 May 2026 11:53:46 -0700 Subject: [PATCH 5/6] chore(ci): split Codecov flag scala -> amber + platform --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f4fee9f9516..bea38561e79 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -248,7 +248,7 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} files: ./**/target/scala-2.13/jacoco/report/jacoco.xml - flags: scala + flags: amber fail_ci_if_error: false platform: @@ -336,14 +336,14 @@ jobs: ./bin/licensing/audit_jar_licenses.py /tmp/dists/${{ matrix.service }}-*/lib || true exit "$check_exit" - name: Upload ${{ matrix.service }} coverage to Codecov - # Same `scala` flag as the amber job so Codecov merges all Scala - # coverage reports into a single view. + # All five matrix entries upload under the `platform` flag so + # Codecov merges them into one platform-stack view. if: always() uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./${{ matrix.service }}/target/scala-2.13/jacoco/report/jacoco.xml - flags: scala + flags: platform fail_ci_if_error: false python: From 423589486c651a7feab7b3d62df7be8a89bedead Mon Sep 17 00:00:00 2001 From: Yicong Huang Date: Sat, 2 May 2026 11:54:42 -0700 Subject: [PATCH 6/6] chore(ci): per-service Codecov flag in platform matrix --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bea38561e79..9bda7c3ddf3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -336,14 +336,14 @@ jobs: ./bin/licensing/audit_jar_licenses.py /tmp/dists/${{ matrix.service }}-*/lib || true exit "$check_exit" - name: Upload ${{ matrix.service }} coverage to Codecov - # All five matrix entries upload under the `platform` flag so - # Codecov merges them into one platform-stack view. + # Per-service flag so each matrix entry has its own Codecov view + # rather than being merged into one umbrella `platform` flag. if: always() uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./${{ matrix.service }}/target/scala-2.13/jacoco/report/jacoco.xml - flags: platform + flags: ${{ matrix.service }} fail_ci_if_error: false python: