diff --git a/.github/workflows/analytics-engine-compat.yml b/.github/workflows/analytics-engine-compat.yml deleted file mode 100644 index e39adb0a311..00000000000 --- a/.github/workflows/analytics-engine-compat.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Analytics Engine Compatibility - -on: - pull_request: - push: - branches-ignore: - - 'backport/**' - - 'dependabot/**' - paths: - - '**/*.java' - - '**gradle*' - - 'integ-test/**' - - '.github/workflows/analytics-engine-compat.yml' - merge_group: - -jobs: - Get-CI-Image-Tag: - uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main - with: - product: opensearch - - analytics-engine-compat: - needs: Get-CI-Image-Tag - runs-on: ubuntu-latest - container: - image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} - options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} - - steps: - - name: Run start commands - run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} - - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Set up JDK 25 - uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 - with: - distribution: 'temurin' - java-version: 25 - - - name: Run analytics-engine compatibility smoke test - run: | - chown -R 1000:1000 `pwd` - su `id -un 1000` -c "./gradlew :integ-test:analyticsEngineCompatIT" diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index af2aaac8264..f68f53993d0 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -1,69 +1,27 @@ -name: PPL lint multi-version validation - -# Multi-version companion to ppl-lint-rule-validation.yml. -# -# The sibling workflow answers "do the OSD PPL lint detectors and THIS engine -# build agree?". It validates one engine: the one built from the PR. That leaves -# the failure mode that actually reaches users unguarded — a lint rule ships to -# everyone, but each user runs it against whatever engine version their cluster -# happens to be. A rule that is correct on main can be a false positive on 3.6 or -# a false negative on 3.7, and nothing notices. -# -# This workflow validates all 12 active detector contracts against released -# engine versions plus the PR build. The aggregate still records the exact -# default-error census separately, but --all-rules makes warning/info omissions -# visible too. -# -# Shape — a per-version matrix of observation legs, then one aggregation: -# -# observe (matrix: 3.6.0, 3.7.0, pr-build) ──▶ aggregate ──▶ drift report -# -# Each leg produces the SAME four artifacts the single-version workflow already -# defines (ppl-grammar-bundle.json, target.json, backend-report.json, -# detector-report.json), so this workflow adds no new producer format — only the -# per-version fan-out and the cross-version comparison. -# -# Released legs run the official distribution image, which bundles the matching -# opensearch-sql plugin (verified against opensearch-build's release manifests), -# so no old branch has to be built. The `pr-build` leg is the same Gradle test -# cluster the sibling workflow uses. -# -# Engine floor: 3.6.0. GET /_plugins/_ppl/_grammar landed in #5162 (`fe95703b5`), -# which is an ancestor of the 3.6 release branch but NOT of 3.5 — a 3.5 leg could -# not export a candidate grammar bundle, so the detector half would have nothing -# to lint against. Raise `ENGINE_VERSIONS` as older versions leave support. -# -# Compatibility differences are collected without interrupting the matrix. -# The aggregate writes the complete table and JSON artifact first, then the -# final step fails this job when a declared-supported rule has drifted. +name: PPL lint multi-surface compatibility + +# Validate the 12 active PPL lint rules on the checked-in OSD fallback grammar +# and on runtime grammar bundles exported by the latest eligible GA engine and +# this SQL pull request. Compatibility differences are data until the aggregate +# job has published the complete 12 x 3 matrix. on: - # Nightly is the primary schedule: the matrix pulls three engine images, so it - # is too slow to sit on every push. - schedule: - - cron: '30 10 * * *' - # Run on PRs that touch the contract corpus or this machinery, where the whole - # point is to see the multi-version effect of the change. pull_request: paths: + - 'build.gradle' - 'integ-test/build.gradle' - 'integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java' - 'integ-test/src/test/resources/ppl-lint/**' - - 'scripts/ppl-lint-rule-validation.sh' - 'scripts/ppl-lint/**' - '.github/workflows/ppl-lint-multiversion-validation.yml' workflow_dispatch: inputs: osd_repo: - description: OSD repository to check out. Defaults to opensearch-project/OpenSearch-Dashboards. + description: OSD repository containing the detector implementation. required: false type: string osd_ref: - description: OSD commit or branch whose detectors are validated. - required: false - type: string - engine_versions: - description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' + description: OSD branch or commit containing the detector implementation. required: false type: string @@ -71,102 +29,90 @@ permissions: contents: read concurrency: - group: ppl-lint-multiversion-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + group: ppl-lint-multi-surface-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true -env: - # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must - # be >= 3.6.0 (the _grammar endpoint floor) and must have a published - # distribution image. - # Latest patch of each line (3.6.0 and 3.7.0 ARE the latest patches today; bump - # them when 3.6.1 / 3.7.1 publish, and never pin `.0` once a newer patch - # exists — that would validate an engine no user runs). - ENGINE_VERSIONS: '["3.6.0","3.7.0"]' - jobs: - # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a - # dependabot bump moves one set of action versions rather than two. Get-CI-Image-Tag: uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main with: product: opensearch - # Resolve the matrix and the OSD target once, so every leg and the aggregate - # step agree on exactly what is being validated. plan: - name: Plan matrix + name: Plan compatibility matrix runs-on: ubuntu-latest outputs: - released: ${{ steps.plan.outputs.released }} - discovery_engine: ${{ steps.plan.outputs.discovery_engine }} - osd_repo: ${{ steps.plan.outputs.osd_repo }} - osd_ref: ${{ steps.plan.outputs.osd_ref }} + released_targets: ${{ steps.outputs.outputs.released_targets }} + osd_repo: ${{ steps.outputs.outputs.osd_repo }} + osd_ref: ${{ steps.outputs.outputs.osd_ref }} + pr_target: ${{ steps.outputs.outputs.pr_target }} steps: - - name: Resolve engine versions and OSD target - id: plan + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Read official OpenSearch GA tags + run: | + set -euo pipefail + git ls-remote --tags --refs https://github.com/opensearch-project/OpenSearch.git \ + > "$RUNNER_TEMP/opensearch-release-tags.txt" + + - name: Resolve target versions and configurations env: - REQUESTED_VERSIONS: ${{ inputs.engine_versions }} - DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} - REQUESTED_REPO: ${{ inputs.osd_repo }} - REQUESTED_REF: ${{ inputs.osd_ref }} - VAR_REPO: ${{ vars.OSD_REPO }} - VAR_REF: ${{ vars.OSD_REF }} + REQUESTED_OSD_REPO: ${{ inputs.osd_repo }} + REQUESTED_OSD_REF: ${{ inputs.osd_ref }} + VARIABLE_OSD_REPO: ${{ vars.OSD_REPO }} + VARIABLE_OSD_REF: ${{ vars.OSD_REF }} run: | set -euo pipefail - released="${REQUESTED_VERSIONS:-$DEFAULT_VERSIONS}" - # Fail loudly on a malformed override rather than silently validating - # an empty matrix (which would look like a pass). - echo "$released" | python3 -c " - import json,sys - v=json.load(sys.stdin) - assert isinstance(v,list) and v, 'engine_versions must be a non-empty JSON array' - for item in v: - assert isinstance(item,str), 'engine_versions entries must be strings' - " - echo "released=$released" >> "$GITHUB_OUTPUT" - - # Discovery runs against ONE engine — the newest released version in the - # matrix. It is a lead-generator, not a version-drift check, so paying for - # a full matrix would multiply cost without adding signal: a false positive - # found on the newest engine is the one users hit soonest, and per-version - # differences are already the enforced corpus's job. - discovery_engine=$(echo "$released" | python3 -c " - import json,sys - v=json.load(sys.stdin) - # Newest by semver, not list order, so a reordered matrix cannot silently - # point discovery at an old engine. - def key(s): - parts=[int(p) for p in s.split('-')[0].split('.') if p.isdigit()] - return parts + [0]*(3-len(parts)) - print(sorted(v,key=key)[-1]) - ") - echo "discovery_engine=$discovery_engine" >> "$GITHUB_OUTPUT" - echo "Discovery engine: \`$discovery_engine\`" >> "$GITHUB_STEP_SUMMARY" - # Same precedence as the sibling workflow: dispatch input, then repo - # variable, then the canonical upstream default. - echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" - echo "osd_ref=${REQUESTED_REF:-${VAR_REF:-main}}" >> "$GITHUB_OUTPUT" - - # One leg per released engine version: run the contract queries against the - # official distribution image (which bundles the matching sql plugin) and - # export that engine's grammar bundle. + osd_repo="${REQUESTED_OSD_REPO:-${VARIABLE_OSD_REPO:-opensearch-project/OpenSearch-Dashboards}}" + osd_ref="${REQUESTED_OSD_REF:-${VARIABLE_OSD_REF:-main}}" + node scripts/ppl-lint/plan-compatibility.mjs \ + --build-file build.gradle \ + --release-tags "$RUNNER_TEMP/opensearch-release-tags.txt" \ + --compiled-version 2.19.6 \ + --sql-sha "$GITHUB_SHA" \ + --osd-repository "$osd_repo" \ + --osd-ref "$osd_ref" \ + --out compatibility-plan.json + + - name: Publish plan outputs + id: outputs + run: | + set -euo pipefail + { + echo "released_targets=$(jq -c '.releasedTargets' compatibility-plan.json)" + echo "osd_repo=$(jq -r '.osd.repository' compatibility-plan.json)" + echo "osd_ref=$(jq -r '.osd.ref' compatibility-plan.json)" + echo "pr_target=$(jq -r '.prTargetVersion' compatibility-plan.json)" + } >> "$GITHUB_OUTPUT" + { + echo '## PPL lint compatibility plan' + echo + echo "- PR target: \`$(jq -r '.prTargetVersion' compatibility-plan.json)\`" + echo "- Latest eligible GA: \`$(jq -r '.latestEligibleGa' compatibility-plan.json)\`" + echo "- OSD: \`$(jq -r '.osd.repository + " @ " + .osd.ref' compatibility-plan.json)\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload compatibility plan + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-compatibility-plan + path: compatibility-plan.json + if-no-files-found: error + observe-released: - name: Observe engine ${{ matrix.version }} + name: Observe engine ${{ matrix.version }} (${{ matrix.label }}) needs: plan runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 35 strategy: fail-fast: false - matrix: - version: ${{ fromJSON(needs.plan.outputs.released) }} + matrix: ${{ fromJSON(needs.plan.outputs.released_targets) }} services: opensearch: image: opensearchproject/opensearch:${{ matrix.version }} env: discovery.type: single-node - # The lint contract only needs the PPL query and grammar endpoints, so - # run without the security plugin: no TLS or credentials to manage, and - # the observed error bodies are the engine's own rather than a proxy's. DISABLE_SECURITY_PLUGIN: 'true' DISABLE_INSTALL_DEMO_CONFIG: 'true' OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g @@ -182,41 +128,53 @@ jobs: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Wait for the engine and confirm its version - id: engine + - name: Confirm released engine identity run: | set -euo pipefail - for i in $(seq 1 40); do - if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi - echo "waiting for engine (${i}/40)..." + for attempt in $(seq 1 40); do + curl -sf http://localhost:9200 > "$RUNNER_TEMP/engine-root.json" && break + echo "waiting for engine (${attempt}/40)..." sleep 5 done - cat /tmp/root.json - reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") - echo "reported=$reported" >> "$GITHUB_OUTPUT" - # A leg mislabeled as another version would attribute drift to the wrong - # engine, so require the image to be what the matrix asked for. + reported=$(jq -r '.version.number' "$RUNNER_TEMP/engine-root.json") case "$reported" in ${{ matrix.version }}*) ;; - *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + *) echo "::error::engine reported $reported; expected ${{ matrix.version }}"; exit 1 ;; esac - # The PPL plugin must actually be present, or every query would "pass" - # by failing identically. curl -sf http://localhost:9200/_cat/plugins | grep -i sql - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: - distribution: 'temurin' + distribution: temurin java-version: 21 - # The same contract oracle the single-version workflow runs, pointed at an - # external cluster instead of a Gradle-managed one. One oracle, many - # engines: a per-version copy would be free to drift from the real check. - - name: Run contract observation against engine ${{ matrix.version }} + - name: Observe backend contracts + env: + EXPORT_RUNTIME_BUNDLE: ${{ matrix.export_runtime_bundle }} run: | set -euo pipefail mkdir -p leg + bundle_args=() + if [ "$EXPORT_RUNTIME_BUNDLE" = 'true' ]; then + bundle_args+=("-Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json") + fi + { + printf './gradlew :integ-test:integTestRemote ' + printf '%q ' \ + '--tests' 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' \ + '-Dtests.rest.cluster=localhost:9200' \ + '-Dtests.cluster=localhost:9200' \ + '-Dtests.clustername=docker-cluster' \ + '-Dppl.lint.schedule=nightly' \ + '-Dppl.lint.observe.only=true' \ + '-Dppl.lint.execution_backend=standard' \ + "-Dppl.lint.sql_sha=$GITHUB_SHA" \ + "-Dppl.lint.report=$(pwd)/leg/backend-report.json" \ + "-Dppl.lint.target=$(pwd)/leg/target.json" \ + "${bundle_args[@]}" + echo + } > leg/backend-command.txt ./gradlew :integ-test:integTestRemote \ --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ -Dtests.rest.cluster=localhost:9200 \ @@ -225,36 +183,37 @@ jobs: -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ -Dppl.lint.execution_backend=standard \ - -Dppl.lint.sql_sha="${GITHUB_SHA}" \ + -Dppl.lint.sql_sha="$GITHUB_SHA" \ -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ - -Dppl.lint.grammar.bundle="$(pwd)/leg/ppl-grammar-bundle.json" \ - -Dppl.lint.target="$(pwd)/leg/target.json" + -Dppl.lint.target="$(pwd)/leg/target.json" \ + "${bundle_args[@]}" - - name: Upload leg artifacts + - name: Upload released observation if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-${{ matrix.version }} + name: ${{ matrix.artifact_name }} path: leg - if-no-files-found: error + if-no-files-found: warn - - name: Upload failure logs + - name: Upload released observation logs if: ${{ failure() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-${{ matrix.version }}-logs - path: integ-test/build/reports/** + name: ${{ matrix.artifact_name }}-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** + if-no-files-found: warn - # The PR's own engine build, so the newest point in the matrix is the code under - # review rather than the last release. Same oracle as the released legs; the only - # difference is a Gradle-managed cluster instead of a published image, which is - # why it cannot just be another matrix entry. observe-pr-build: - name: Observe engine pr-build - needs: Get-CI-Image-Tag + name: Observe engine pr-build (runtime) + needs: + - Get-CI-Image-Tag + - plan runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 35 container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} @@ -268,75 +227,67 @@ jobs: - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: - distribution: 'temurin' + distribution: temurin java-version: 21 - # Observe-only here too, so this leg reports what the PR engine does rather - # than duplicating the sibling workflow's assertions. The sibling workflow - # remains the enforcing single-version check. - - name: Run contract observation against the PR build + - name: Observe backend contracts run: | set -euo pipefail mkdir -p leg chown -R 1000:1000 "$(pwd)" - su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ - --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ - -Dppl.lint.schedule=nightly \ - -Dppl.lint.observe.only=true \ - -Dppl.lint.execution_backend=standard \ - -Dppl.lint.sql_sha=${GITHUB_SHA} \ - -Dppl.lint.report=$(pwd)/leg/backend-report.json \ - -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ - -Dppl.lint.target=$(pwd)/leg/target.json" + command="./gradlew :integ-test:integTest --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true -Dppl.lint.execution_backend=standard -Dppl.lint.sql_sha=$GITHUB_SHA -Dppl.lint.report=$(pwd)/leg/backend-report.json -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json -Dppl.lint.target=$(pwd)/leg/target.json" + printf '%s\n' "$command" > leg/backend-command.txt + su "$(id -un 1000)" -c "$command" - - name: Upload leg artifacts + - name: Upload PR-build observation if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-pr-build + name: ppl-lint-observation-pr-build-runtime path: leg - if-no-files-found: error + if-no-files-found: warn - - name: Upload failure logs + - name: Upload PR-build observation logs if: ${{ failure() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: ppl-lint-leg-pr-build-logs + name: ppl-lint-observation-pr-build-runtime-logs path: | integ-test/build/reports/** integ-test/build/test-results/** integ-test/build/testclusters/*/logs/* + if-no-files-found: warn - # Lint each engine's exported grammar with the OSD detectors. Separate from the - # observation legs because OSD needs a newer Node/glibc than the engine image - # provides, and because one bootstrap can serve every leg. - # - # `always()` so a single broken leg still yields a report for the others: a - # partial matrix must be visibly partial, not silently absent. The aggregate - # step fails if NO leg produced a report. - detect: + aggregate: name: Aggregate rule compatibility + if: ${{ always() && needs.plan.result == 'success' }} needs: - plan - observe-released - observe-pr-build - if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest - timeout-minutes: 40 - outputs: - osd_sha: ${{ steps.osd-rev.outputs.sha }} + timeout-minutes: 45 steps: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Download all leg artifacts + - name: Download compatibility plan + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + name: ppl-lint-compatibility-plan + path: plan + + - name: Download available observations + continue-on-error: true uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 with: - pattern: ppl-lint-leg-* + pattern: ppl-lint-observation-* path: legs - name: Checkout OpenSearch-Dashboards + id: osd-checkout + continue-on-error: true uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: repository: ${{ needs.plan.outputs.osd_repo }} @@ -344,18 +295,27 @@ jobs: path: .ci/OpenSearch-Dashboards - name: Record OSD revision - id: osd-rev + id: osd-revision + if: ${{ always() }} run: | - sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) - echo "sha=$sha" >> "$GITHUB_OUTPUT" - echo "OSD revision: \`$sha\` (${{ needs.plan.outputs.osd_repo }} @ \`${{ needs.plan.outputs.osd_ref }}\`)" >> "$GITHUB_STEP_SUMMARY" + if sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD 2>/dev/null); then + echo "available=true" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "sha=" >> "$GITHUB_OUTPUT" + fi - name: Set up Node from OSD .nvmrc + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 with: node-version-file: .ci/OpenSearch-Dashboards/.nvmrc - name: Pin Yarn from OSD engines + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true working-directory: .ci/OpenSearch-Dashboards run: | yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") @@ -363,123 +323,123 @@ jobs: npm install -g "yarn@${yarn_version}" - name: Cache OSD Yarn dependencies + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 with: - path: | - ~/.cache/yarn + path: ~/.cache/yarn key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-osd-yarn- + restore-keys: ${{ runner.os }}-osd-yarn- - - name: Bootstrap OpenSearch-Dashboards + - name: Bootstrap OpenSearch-Dashboards once + id: osd-bootstrap + if: ${{ steps.osd-revision.outputs.available == 'true' }} + continue-on-error: true working-directory: .ci/OpenSearch-Dashboards run: | - for i in 1 2 3; do + for attempt in 1 2 3; do yarn osd bootstrap && exit 0 - echo "Bootstrap attempt $i failed, retrying in 10s..." + echo "Bootstrap attempt $attempt failed; retrying in 10 seconds." sleep 10 done exit 1 - # One detector pass per leg, each against THAT engine's grammar bundle. The - # runner is the same SQL-owned script the single-version workflow uses, so - # the detector half cannot drift between the two checks. - - name: Run detectors against every engine grammar - working-directory: .ci/OpenSearch-Dashboards + - name: Run applicable detector passes + if: ${{ steps.osd-bootstrap.outcome == 'success' }} + continue-on-error: true run: | - set -euo pipefail - shopt -s nullglob - legs=("$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*) - if [ ${#legs[@]} -eq 0 ]; then - echo "::error::no leg artifacts were downloaded; nothing to validate." - exit 1 - fi - for leg in "${legs[@]}"; do - version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') - if [ ! -f "$leg/ppl-grammar-bundle.json" ]; then - # Skip the log-only artifacts an observation failure may have uploaded. - echo "skipping $leg (no runtime grammar bundle)" + set -uo pipefail + while IFS= read -r configuration; do + id=$(jq -r '.id' <<< "$configuration") + artifact=$(jq -r '.artifactName' <<< "$configuration") + surface=$(jq -r '.surface' <<< "$configuration") + engine_mode=$(jq -r '.engineMode' <<< "$configuration") + leg="$GITHUB_WORKSPACE/legs/$artifact" + mkdir -p "$leg" + if [ ! -s "$leg/target.json" ] || [ ! -s "$leg/backend-report.json" ]; then + echo "Skipping detector pass for $id: backend evidence is incomplete." continue fi - echo "=== detectors vs engine $version (runtime-bundle surface) ===" - env PPL_LINT_SURFACE=runtime-bundle \ - PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json" \ - PPL_LINT_OBSERVE_ONLY=1 \ - PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ - PPL_LINT_SCHEDULE=nightly \ - PPL_LINT_INCLUDE_DORMANT=1 \ - PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ - PPL_LINT_REPORT="$leg/detector-report.json" \ - node -r ./src/setup_node_env \ - "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ - > "$leg/detector.log" 2>&1 || true - # A per-leg non-zero exit is EXPECTED when that engine disagrees with - # the pinned expectation — that is the drift this workflow exists to - # report, and the aggregate step below is what classifies it. Only a - # missing report means the runner itself broke. - if [ ! -f "$leg/detector-report.json" ]; then - echo "::error::detector runner produced no report for engine $version" - tail -50 "$leg/detector.log" || true - exit 1 + + if [ "$surface" = 'compiled-simplified' ]; then + grammar_root=".ci/OpenSearch-Dashboards/packages/osd-antlr-grammar/src/opensearch_ppl_simplified" + grammar_hash=$( + find "$grammar_root" -type f -print0 | + sort -z | + xargs -0 sha256sum | + sha256sum | + awk '{print "sha256:" $1}' + ) + jq --arg hash "$grammar_hash" \ + '.grammarHash = $hash | .grammarBundle = ""' \ + "$leg/target.json" > "$leg/detector-target.json" + bundle='' + else + cp "$leg/target.json" "$leg/detector-target.json" + bundle="$leg/ppl-grammar-bundle.json" + if [ ! -s "$bundle" ]; then + echo "Skipping detector pass for $id: runtime grammar bundle is missing." + continue + fi fi - tail -5 "$leg/detector.log" || true - done - # Compare every engine version against every other and against the pinned - # contracts, then print the remediation report. - - name: Aggregate drift across engine versions - id: aggregate - env: - RELEASED: ${{ needs.plan.outputs.released }} + { + printf 'cd %q && env ' "$GITHUB_WORKSPACE/.ci/OpenSearch-Dashboards" + printf 'PPL_LINT_SURFACE=%q ' "$surface" + printf 'PPL_LINT_ENGINE_MODE=%q ' "$engine_mode" + printf 'PPL_LINT_APPLICABLE_ONLY=1 ' + printf 'PPL_LINT_GRAMMAR_BUNDLE=%q ' "$bundle" + printf 'PPL_LINT_TARGET_MANIFEST=%q ' "$leg/detector-target.json" + printf 'PPL_LINT_BACKEND_REPORT=%q ' "$leg/backend-report.json" + printf 'PPL_LINT_CONTRACT_DIR=%q ' "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" + printf 'PPL_LINT_SCHEDULE=nightly PPL_LINT_OBSERVE_ONLY=1 ' + printf 'PPL_LINT_REPORT=%q ' "$leg/detector-report.json" + printf 'node -r ./src/setup_node_env %q\n' \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + } > "$leg/detector-command.txt" + + ( + cd .ci/OpenSearch-Dashboards + env \ + PPL_LINT_SURFACE="$surface" \ + PPL_LINT_ENGINE_MODE="$engine_mode" \ + PPL_LINT_APPLICABLE_ONLY=1 \ + PPL_LINT_GRAMMAR_BUNDLE="$bundle" \ + PPL_LINT_TARGET_MANIFEST="$leg/detector-target.json" \ + PPL_LINT_BACKEND_REPORT="$leg/backend-report.json" \ + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_OBSERVE_ONLY=1 \ + PPL_LINT_REPORT="$leg/detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" + ) > "$leg/detector.log" 2>&1 || true + tail -10 "$leg/detector.log" || true + done < <(jq -c '.configurations[]' plan/compatibility-plan.json) + + - name: Aggregate every planned rule and configuration + id: compatibility + if: ${{ always() }} run: | - set -euo pipefail - shopt -s nullglob - args=() - present=() - for leg in "$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*; do - [ -f "$leg/detector-report.json" ] || continue - version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') - args+=(--leg "$version=$leg") - present+=("$version") - done - if [ ${#args[@]} -eq 0 ]; then - echo "::error::no complete legs to aggregate." - exit 1 - fi - # Every version the plan asked for must have produced a leg. Aggregating - # only the survivors would report "PASS: agrees with all N versions" over - # a matrix that silently lost one — the exact vacuous pass this workflow - # exists to prevent. A dead leg is a failure, not a smaller matrix. - missing=() - for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") pr-build; do - found=no - for have in "${present[@]}"; do - [ "$have" = "$want" ] && found=yes && break - done - [ "$found" = yes ] || missing+=("$want") - done - if [ ${#missing[@]} -gt 0 ]; then - echo "::error::planned engine leg(s) produced no report: ${missing[*]}. Check those observe jobs; the matrix is incomplete so its result would be misleading." - exit 1 - fi set +e - node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ - --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ - --out "$GITHUB_WORKSPACE/drift-report.json" \ - --summary "$GITHUB_STEP_SUMMARY" \ - --all-rules \ - "${args[@]}" - aggregate_exit=$? - set -e - echo "exit_code=$aggregate_exit" >> "$GITHUB_OUTPUT" + node scripts/ppl-lint/aggregate-compatibility.mjs \ + --plan plan/compatibility-plan.json \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --artifacts legs \ + --osd-sha "${{ steps.osd-revision.outputs.sha }}" \ + --out drift-report.json \ + --summary "$GITHUB_STEP_SUMMARY" + result=$? + echo "exit_code=$result" >> "$GITHUB_OUTPUT" + exit 0 - name: Upload drift report if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: ppl-lint-multiversion-drift - if-no-files-found: error path: drift-report.json + if-no-files-found: error - name: Upload compatibility evidence if: ${{ always() }} @@ -487,277 +447,45 @@ jobs: uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: name: ppl-lint-multiversion-evidence - if-no-files-found: warn path: | + plan/compatibility-plan.json + legs/**/backend-report.json legs/**/detector-report.json - legs/**/detector.log legs/**/target.json + legs/**/detector-target.json + legs/**/*-command.txt + legs/**/detector.log + if-no-files-found: warn - - name: Fail after publishing compatibility drift + - name: Fail after publishing compatibility results if: ${{ always() }} env: - AGGREGATE_EXIT: ${{ steps.aggregate.outputs.exit_code }} - RELEASED: ${{ needs.plan.outputs.released }} + AGGREGATE_EXIT: ${{ steps.compatibility.outputs.exit_code }} run: | set -euo pipefail - if [ -z "$AGGREGATE_EXIT" ]; then - echo "::error::aggregation did not complete" + if [ ! -s drift-report.json ]; then + echo "::error::drift-report.json was not published" exit 1 fi - expected_legs=$(echo "$RELEASED" | jq 'length + 1') - expected_rules=$(jq -c \ - '[.contracts[] | sub("\\.spec\\.json$"; "")] | sort' \ - integ-test/src/test/resources/ppl-lint/contracts/manifest.json) - if [ ! -s drift-report.json ] || ! jq -e \ - --argjson expected_legs "$expected_legs" \ - --argjson expected_rules "$expected_rules" \ - ' - type == "object" and - ($expected_rules | length) == 12 and - ($expected_rules | index("command-suggestion") | not) and - (.legs | type == "array" and length == $expected_legs) and - (.matrix | type == "array") and - (.matrix | length) == (12 * $expected_legs) and - (.matrix | map(.ruleId) | unique | sort) == $expected_rules and - (.matrix | map(.legKey) | unique | length) == $expected_legs and - (.matrix | map(.legKey) | unique | sort) == - (.legs | map(.key) | unique | sort) and - ([.matrix[] | [.ruleId, .legKey]] | unique | length) == - (12 * $expected_legs) - ' drift-report.json > /dev/null; then - echo "::error::aggregation did not produce the complete 12-rule compatibility matrix" + jq -e ' + .schemaVersion == 3 and + .inventory.ruleCount == 12 and + (.inventory.ruleIds | length) == 12 and + (.configurations | length) == 3 and + (.matrix | length) == 36 and + .result.cellCount == 36 and + ( + .result.compatible + .result.notApplicable + + .result.drift + .result.inconclusive + ) == 36 + ' drift-report.json > /dev/null + if [ -z "$AGGREGATE_EXIT" ]; then + echo "::error::compatibility aggregation did not complete" exit 1 fi if [ "$AGGREGATE_EXIT" -ne 0 ]; then - echo "::error::rule compatibility validation failed; see the table above and the ppl-lint-multiversion-drift artifact" + drift=$(jq -r '.result.drift' drift-report.json) + inconclusive=$(jq -r '.result.inconclusive' drift-report.json) + echo "::error::rule compatibility validation failed after artifact publication: $drift drift, $inconclusive inconclusive" exit "$AGGREGATE_EXIT" fi - - # Discovery: harvest queries from OSD's own lint tests, run both halves over them, - # and report detector/engine disagreements as LEADS. - # - # Why this is separate from `detect`, and why it can never fail the build: - # - # The enforced corpus is hand-pinned — every expectation is a reviewed claim, which - # is what lets a mismatch red the build. That corpus is also small (about one - # trigger per rule), and `classifyRelaxationScope` needs SEVERAL triggers per rule - # to tell a FULL engine fix (version-scope the rule away) from a PARTIAL one - # (narrow the detector). Those need opposite actions, so with one trigger the - # advice can be confidently wrong. - # - # This job supplies that trigger variety from queries OSD's own detector authors - # already wrote. It pins NOTHING: roles are derived from real detector output and - # the engine supplies the other half, so no expectation is ever auto-generated. - # An auto-derived expectation could only confirm current behavior — locking in - # whatever the detector does today, bugs included. - # - # `continue-on-error` AND a zero exit from the labeler: a finding here is a lead to - # investigate, not a proven defect, and blocking unrelated PRs on an auto-generated - # guess would poison the whole check's credibility. - discovery: - name: Discovery corpus (harvested, not enforced) - # Only `plan`, for the OSD target and the engine version. Deliberately NOT the - # observe legs: discovery runs its own engine and harvests its own queries, so - # depending on them would idle this job behind ~30 minutes of matrix work it - # never reads, and a failed leg would block a report that does not need it. - needs: plan - continue-on-error: true - runs-on: ubuntu-latest - timeout-minutes: 40 - services: - opensearch: - image: opensearchproject/opensearch:${{ needs.plan.outputs.discovery_engine }} - env: - discovery.type: single-node - DISABLE_SECURITY_PLUGIN: 'true' - DISABLE_INSTALL_DEMO_CONFIG: 'true' - OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g - ports: - - 9200:9200 - options: >- - --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" - --health-interval 15s - --health-timeout 10s - --health-retries 20 - --health-start-period 60s - steps: - - name: Checkout SQL pull request - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Checkout OpenSearch-Dashboards - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - repository: ${{ needs.plan.outputs.osd_repo }} - ref: ${{ needs.plan.outputs.osd_ref }} - path: .ci/OpenSearch-Dashboards - - - name: Set up Node from OSD .nvmrc - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 - with: - node-version-file: .ci/OpenSearch-Dashboards/.nvmrc - - - name: Pin Yarn from OSD engines - working-directory: .ci/OpenSearch-Dashboards - run: | - yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") - yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') - npm install -g "yarn@${yarn_version}" - - - name: Cache OSD Yarn dependencies - uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 - with: - path: | - ~/.cache/yarn - key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-osd-yarn- - - - name: Bootstrap OpenSearch-Dashboards - working-directory: .ci/OpenSearch-Dashboards - run: | - for i in 1 2 3; do - yarn osd bootstrap && exit 0 - echo "Bootstrap attempt $i failed, retrying in 10s..." - sleep 10 - done - exit 1 - - # The rule id list comes from the OSD catalog being validated, not a hardcoded - # copy: attribution keys off `describe('')` titles, so a stale list - # would silently stop harvesting queries for any newly added rule. - - name: Harvest the discovery corpus from OSD's lint tests - run: | - set -euo pipefail - node -e " - const c = require('./.ci/OpenSearch-Dashboards/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); - process.stdout.write(JSON.stringify(c.map((r) => r.id))); - " > /tmp/catalog-rules.json - node scripts/ppl-lint/harvest-queries.mjs \ - --osd .ci/OpenSearch-Dashboards \ - --catalog-rules @/tmp/catalog-rules.json \ - --index opensearch-sql_test_index_account \ - --out "$GITHUB_WORKSPACE/discovery-corpus.json" \ - --specs-out "$GITHUB_WORKSPACE/discovery-specs" - - # Seed the one index every harvested query was rewritten onto. Without it the - # engine rejects everything with IndexNotFoundException — which the labeler - # would correctly suppress as uninformative, yielding a run that reports - # nothing at all. - - name: Seed the fixture index - run: | - set -euo pipefail - for i in $(seq 1 40); do - curl -sf http://localhost:9200 > /dev/null && break - echo "waiting for engine (${i}/40)..." - sleep 5 - done - curl -sf -X PUT "http://localhost:9200/opensearch-sql_test_index_account" \ - -H 'content-type: application/json' -d '{ - "mappings": { "properties": { - "account_number": { "type": "long" }, - "balance": { "type": "long" }, - "age": { "type": "integer" }, - "status": { "type": "keyword" }, - "firstname": { "type": "text" }, - "lastname": { "type": "text" }, - "msg": { "type": "text" }, - "body": { "type": "text" }, - "raw": { "type": "object", "enabled": false } - } } - }' - curl -sf -X POST "http://localhost:9200/opensearch-sql_test_index_account/_doc?refresh=true" \ - -H 'content-type: application/json' \ - -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' - - # Export this engine's grammar bundle. Discovery only runs against the runtime - # grammar surface, so an unavailable endpoint skips the detector pass. - - name: Export the engine grammar bundle - id: bundle - run: | - set -uo pipefail - if curl -sf --max-time 60 "http://localhost:9200/_plugins/_ppl/_grammar" \ - -o "$GITHUB_WORKSPACE/discovery-bundle.json"; then - hash=$(python3 -c " - import json - print(json.load(open('$GITHUB_WORKSPACE/discovery-bundle.json')).get('grammarHash','')) - ") - python3 -c " - import json - json.dump({'schemaVersion': 2, - 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', - 'grammarHash': '$hash', - 'grammarBundle': 'discovery-bundle.json', - 'executionBackend': 'standard', - 'storage': 'lucene', - 'shardCount': 1}, - open('$GITHUB_WORKSPACE/discovery-target.json','w')) - " - echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" - else - echo "::warning::_grammar export failed; skipping discovery detector pass." - echo "surface=unavailable" >> "$GITHUB_OUTPUT" - fi - - - name: Run the detectors over the discovery corpus - working-directory: .ci/OpenSearch-Dashboards - env: - SURFACE: ${{ steps.bundle.outputs.surface }} - run: | - set -uo pipefail - if [ "$SURFACE" != 'runtime-bundle' ]; then - echo "Discovery detector pass skipped: runtime grammar bundle unavailable." - exit 0 - fi - # A non-zero exit is EXPECTED and ignored: the generated specs carry - # placeholder expectations, so the runner reports a "failure" for every - # query whose real diagnostic count differs. Only the report is read. - env PPL_LINT_DISCOVERY=1 \ - PPL_LINT_SURFACE=runtime-bundle \ - PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" \ - PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json" \ - PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ - PPL_LINT_SCHEDULE=nightly \ - PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ - node -r ./src/setup_node_env \ - "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ - > "$GITHUB_WORKSPACE/discovery-detector.log" 2>&1 || true - if [ ! -f "$GITHUB_WORKSPACE/discovery-detector-report.json" ]; then - echo "::warning::the detector runner produced no discovery report; skipping." - tail -50 "$GITHUB_WORKSPACE/discovery-detector.log" || true - fi - - - name: Probe the engine with the discovery corpus - run: | - set -euo pipefail - node scripts/ppl-lint/probe-discovery-backend.mjs \ - --corpus discovery-corpus.json \ - --endpoint http://localhost:9200 \ - --out discovery-backend-report.json - - - name: Label and report - run: | - set -euo pipefail - if [ ! -f discovery-detector-report.json ]; then - echo "::warning::no detector report; nothing to label." - exit 0 - fi - node scripts/ppl-lint/label-discovery.mjs \ - --corpus discovery-corpus.json \ - --detector discovery-detector-report.json \ - --backend discovery-backend-report.json \ - --version "${{ needs.plan.outputs.discovery_engine }}" \ - --out discovery-findings.json \ - --summary "$GITHUB_STEP_SUMMARY" - - - name: Upload discovery artifacts - if: ${{ always() }} - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - continue-on-error: true - with: - name: ppl-lint-discovery - path: | - discovery-corpus.json - discovery-findings.json - discovery-detector-report.json - discovery-backend-report.json - discovery-detector.log diff --git a/docs/dev/ppl-lint-analytics-engine-ci-validation.md b/docs/dev/ppl-lint-analytics-engine-ci-validation.md index ba7632b2d54..81c77f0b771 100644 --- a/docs/dev/ppl-lint-analytics-engine-ci-validation.md +++ b/docs/dev/ppl-lint-analytics-engine-ci-validation.md @@ -89,13 +89,13 @@ The repository already contains most of the required test infrastructure: `LogicalTableScan(table=[[opensearch,` and not `CalciteLogicalIndexScan`. -### 2.3 Gap in the existing analytics workflow +### 2.3 Gap in the retired analytics workflow -`.github/workflows/analytics-engine-compat.yml` runs only -`AnalyticsEngineCompatIT`. Its purpose is plugin coexistence. Its PPL assertion -uses the `rest` row source, which is explicitly excluded from analytics -routing. The workflow can therefore pass without executing a PPL query through -DataFusion. +The removed `.github/workflows/analytics-engine-compat.yml` ran only +`AnalyticsEngineCompatIT`. Its purpose was plugin coexistence. Its PPL +assertion used the `rest` row source, which is explicitly excluded from +analytics routing. The workflow could therefore pass without executing a PPL +query through DataFusion. The `analyticsEngineCompat` cluster is also intentionally smaller than the stack required for real analytics execution. It does not install the composite @@ -763,8 +763,8 @@ After the required lane is stable, evaluate: - Matching released analytics stacks. - A scheduled three-shard analytics leg. - Analytics execution for the discovery corpus. -- Consolidating or retiring redundant parts of - `analytics-engine-compat.yml`. +- Reintroducing a standalone analytics workflow only if it validates the + routed analytics contract described here. These are separate changes and are not prerequisites for initial enforcement. diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 660a88c09af..769435da88b 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -114,7 +114,10 @@ writes `detector-report.json`. | --- | --- | | `PPL_LINT_CONTRACT_DIR` | directory of `*.spec.json` + `manifest.json` | | `PPL_LINT_SCHEDULE` | `pr` or `nightly` | -| `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required; no compiled fallback) | +| `PPL_LINT_SURFACE` | `runtime-bundle` (default) or explicit `compiled-simplified` | +| `PPL_LINT_ENGINE_MODE` | optional `calcite` or `legacy` identity for compatibility filtering/context | +| `PPL_LINT_APPLICABLE_ONLY` | `1` omits contracts excluded by surface, version, or engine mode | +| `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required on the runtime surface) | | `PPL_LINT_TARGET_MANIFEST` | schema-v2 `target.json` (engine, grammar, execution backend, and storage identity) | | `PPL_LINT_BACKEND_REPORT` | `backend-report.json` (enables the differential) | | `PPL_LINT_REPORT` | where to write `detector-report.json` | @@ -223,40 +226,49 @@ status, not blocking behavior**. The check above validates **one** engine: the build from the PR. But a lint rule ships to every user, and each user's cluster is on whatever version they run. A -rule that is correct on `main` can be a false positive on 3.6 or a false negative -on 3.7, and the single-version check cannot see it. +rule that is correct on `main` can be a false positive or false negative on a +released cluster, and the single-version check cannot see it. [`ppl-lint-multiversion-validation.yml`](../../.github/workflows/ppl-lint-multiversion-validation.yml) -validates every active shipping detector against several engine versions at -once, and reports **what to change in the linter** when one disagrees. +validates every active shipping detector against three planned configurations +and reports **what to change** when one disagrees: + +- the OSD compiled-simplified fallback grammar against OpenSearch 2.19.6; +- the runtime grammar exported by the highest official GA release at or below + the normalized SQL PR target; +- the runtime grammar exported by the SQL PR build. ``` -observe (matrix: released 3.6/3.7 images + standard pr-build) - └── each leg exports the same 4 artifacts as the single-version check -aggregate rule compatibility (one OSD bootstrap, one detector pass per runtime grammar) - └── aggregate-versions.mjs → drift-report.json + remediation report +plan configurations + ├── observe 2.19.6 backend (compiled comparison) + ├── observe latest eligible GA + export runtime grammar + └── observe PR build + export runtime grammar +aggregate rule compatibility (one OSD bootstrap, three detector passes) + └── aggregate-compatibility.mjs → 36-cell schema-v3 report ``` -Released legs run the official `opensearchproject/opensearch:` image, -which bundles the matching `opensearch-sql` plugin, so no old branch is built. The -`pr-build` leg is the same Gradle test cluster the single-version check uses. The -legs run the **same** contract oracle (`PplLintRuleValidationIT`) with +Released legs run official `opensearchproject/opensearch:` images, +which bundle the matching `opensearch-sql` plugin, so no old branch is built. +The plan reads the default `opensearch.version` from `build.gradle`, removes its +prerelease/build suffix, and selects the highest exact-semver OpenSearch tag at +or below it. The `pr-build` leg uses a Gradle test cluster. All legs run the +**same** contract oracle (`PplLintRuleValidationIT`) with `-Dppl.lint.observe.only=true`, which records real behavior instead of asserting against expectations — on an older engine a mismatch is the signal being collected, not a broken run. -**Engine floor: 3.6.0.** -`GET /_plugins/_ppl/_grammar` landed in #5162, which is an ancestor of 3.6 but not -3.5, so a 3.5 leg cannot export a grammar bundle for the detectors to lint against. - -This workflow intentionally excludes the compiled-simplified surface and -analytics engine. Those dimensions do not share the stable runtime-bundle -contract being compared here. +The 2.19.6 leg does not request a runtime bundle. Its backend observations are +joined to a detector pass over OSD's checked-in simplified grammar. Rules with +`grammarSurface: runtime-bundle` are `n/a (surface)` there; rules outside their +`wiring.appliesTo` version range are `n/a (version)`. Surface takes precedence +when both exclusions apply. Analytics, syntax-channel, dormant-rule, discovery, +and AI action tests are not part of this workflow. -Observation jobs do not fail on compatibility differences. The final +Observation jobs do not fail on compatibility differences. A failed or missing +observation remains a planned column and becomes `inconclusive`. The final `Aggregate rule compatibility` job writes the complete expected-versus-actual -table and `drift-report.json`, uploads them, and then fails when a rule drifts on -a version declared by its `wiring.appliesTo` scope. +table and `drift-report.json`, uploads both report and evidence, and only then +fails for drift or inconclusive in-scope cells. ### What a drift report tells you @@ -264,20 +276,16 @@ Every finding names a drift class, the evidence, and one remediation action: | Action | When | What you change | | --- | --- | --- | -| `version-scope-rule` | the engine relaxed (or never had) the behavior on some versions | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` — or `enabled: false` if no supported engine rejects it any more | +| `scope-rule-version` | every contracted trigger is now accepted and controls prove support | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` | +| `narrow-detector` | only some contracted triggers are now accepted | keep the version in scope and narrow the detector to invalid forms | | `update-detector` | the detector regressed, went too broad, or its grammar anchor was renamed | the rule's detector `.ts` (named in the finding) | | `update-contract` | the linter is right and only the pinned expectation is stale | the `expectations[]` entry for that version | -| `align-execution-backends` | standard and analytics disagree for the same SQL version and grammar | reconcile the detector with both routes or add a reliable backend signal to OSD | - -Drift classes: `grammar-rule-missing` (a parser rule the detector walks was -renamed or removed — the finding names the closest current rule names), -`engine-relaxed` / `engine-partially-relaxed` / `engine-tightened` (the engine's -verdict flipped), `engine-message-changed` (same verdict, reworded error), -`detector-silent` / `detector-noisy` (false negative / false positive), -`version-scope-too-narrow` (the engine rejects but the rule is scoped away from -that version, so users see no diagnostic), `execution-backend-divergence` (same -version, different route verdict), and `severity-mismatch`. Backend divergence -never recommends changing a version range. +| `fix-test-leg` | a detector/backend row or target identity is missing or errored | repair or rerun the test leg before changing product behavior | + +The schema-v3 report consolidates query symptoms into four rule/configuration +classifications: `detector-regression`, `full-engine-relaxation`, +`partial-engine-relaxation`, and `contract-drift`. Missing or errored evidence +is `inconclusive`, not a compatibility classification. #### Full vs partial relaxation: scope the rule, or narrow the detector? @@ -285,10 +293,12 @@ When an engine starts accepting a query a rule flags, the fix depends on a quest a single query cannot answer: is the behavior **fully** gone on that version, or only **partially**? -- **Every trigger relaxed** → `engine-relaxed`, action `version-scope-rule`. Nothing +- **Every trigger relaxed** → `full-engine-relaxation`, action + `scope-rule-version`. Nothing the rule claims is still true on that engine, so bound it with `maxVersion`. -- **Some triggers relaxed, others still rejected** → `engine-partially-relaxed`, - action `update-detector`. The engine fixed *part* of the condition. Scoping the +- **Some triggers relaxed, others still rejected** → + `partial-engine-relaxation`, action `narrow-detector`. The engine fixed *part* + of the condition. Scoping the rule away here would drop the diagnostics that are still correct, converting a partial engine fix into a shipped **false negative**. Narrow the detector so it stops matching the now-valid shapes while still flagging the rest. @@ -299,24 +309,16 @@ supersedes the per-query ones. A trigger with no verdict is counted as neither treating it as "still rejects" would let a timed-out leg masquerade as a partial fix and send someone to narrow a healthy detector. -The evidence always states the tally (`2 of 3 observed trigger(s) relaxed`), and a -rule with only one pinned trigger gets an explicit warning that a "fully relaxed" -verdict rests on a single observation. That is the gap the discovery corpus below -closes. - -Three hard guards keep the check from passing vacuously. The shipping census is -also recorded, but remains report-only until the paired OSD default-alignment -change lands: - -- A rule that is default-error in OSD's catalog but has no contract file is - reported in the shipping census. The detector runner records the catalog's - default-error census in `detector-report.json`, and the aggregate step compares - it against `manifest.defaultError`. This becomes blocking when census - enforcement is enabled after OSD defaults are aligned. -- A leg whose artifacts are missing is a hard failure, never a silently dropped - version. The aggregate step also checks that every version the plan asked for - produced a report, so a dead observe job cannot shrink the matrix into a green - "agrees with all N versions". +The evidence always states the contracted, accepted, rejected, and missing +trigger tally. A one-trigger rule can be fully relaxed when that trigger and its +controls produce complete evidence. + +Four hard guards keep the check from passing vacuously: + +- The active manifest must contain exactly the approved 12 rule IDs. +- A leg whose artifacts are missing remains in the matrix as a complete + `inconclusive` column. A dead observe job cannot shrink the matrix into a green + result. - A case with no engine verdict (a transport failure, recorded by the IT as `outcome: "error"`) is **not** read as acceptance. Coercing it would report a timeout as an engine that now accepts the query — and advise disabling a @@ -327,81 +329,46 @@ change lands: — it proved nothing. Inconclusive findings say "check that leg's logs and re-run", never "edit the rule", because the linter is not what went wrong. -A rule that is out of scope on an engine (`appliesTo` excludes it) and that the -engine also accepts is reported as `n/a (out of scope)`, not as drift — that is -the version window working. But if the engine *rejects* the trigger there, it is -`version-scope-too-narrow`. +A rule that is out of scope for a surface, version, or engine mode is not +executed or compared. Its cell is `n/a` with the corresponding reason and never +blocks the job. ### Where a failure shows up in the GitHub UI -Every finding is emitted twice, because the run page and the diff are two -different places a developer looks: - -1. **Annotations** (top of the run page, and inline on the file in *Files - changed* when the contract is part of the PR's diff). Each carries the drift - class, the rule, the engine version, and the one-line action. An - `update-contract` finding anchors on the exact `expectations[]` entry whose - `version` range produced it — not the top of the file — so the drift appears on - the line that caused it. Rule-wide findings (a renamed grammar rule) anchor on - the contract's `ruleId` instead. -2. **The job summary** — the rule × version table plus the full grouped - remediation report, which stays the authoritative account. - -The required single-version lane follows the same rule: frontend and backend -failures with a `[rule/query]` identity anchor on that contract's `ruleId`. -An individual detector/query execution error is recorded as an `error` row and -does not stop the remaining contracts from running or prevent -`detector-report.json` from being uploaded. The required check still fails after -the complete report is written, with the failing rule/query named directly. -Shipping-census findings anchor on `manifest.json` and remain report-only until -the paired OSD default-alignment change lands. Artifact and job failures without -a trustworthy repository location remain file-less rather than pointing at a -guessed line. - -Without the annotations the only thing above the summary is `Process completed -with exit code 1`, so the natural next click lands in raw job logs rather than the -remediation. Severity is not cosmetic: - -| Finding | Level | Why | -| --- | --- | --- | -| enforced drift, coverage hole | `error` | an active shipping rule disagrees with a supported engine; aggregation writes the report and then fails | -| non-enforced drift | `warning` | reported, but it does not block | -| `inconclusive` | `warning` | "we could not check" is a leg problem, not evidence that a rule is wrong | -| unvalidated default-error rule | `error` (no file) | the edit goes in `manifest.json`, not a contract | - -A line number is emitted only when it is unambiguous. If a contract pins the same -version range twice, or the range cannot be found, the annotation carries the file -and no line — a wrong line sends the reader to edit the wrong expectation, which -is worse than making them find it. +The `Aggregate rule compatibility` step summary is the primary interface. It +always contains all 12 rows and all three columns, followed by blocking findings +and remediation. `ppl-lint-multiversion-drift/drift-report.json` carries the +complete schema-v3 matrix and query cases; `ppl-lint-multiversion-evidence` +carries target identities, detector/backend reports, logs, and reproduction +commands. The final step fails only after both uploads have run. ### Running the multi-version check locally -Each leg needs a reachable cluster. Point the observe step at any running engine: +Use the planner with an exact-semver tag list, then point the aggregator at the +three artifact directories produced by backend and detector runs: ```bash -# Observe one engine (repeat per version into its own leg dir). -mkdir -p legs/3.7.0 -./gradlew :integ-test:integTestRemote \ - --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ - -Dtests.rest.cluster=localhost:9200 \ - -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true \ - -Dppl.lint.report=$PWD/legs/3.7.0/backend-report.json \ - -Dppl.lint.grammar.bundle=$PWD/legs/3.7.0/ppl-grammar-bundle.json \ - -Dppl.lint.target=$PWD/legs/3.7.0/target.json - -# Lint each leg's grammar from an OSD checkout (writes detector-report.json), -# then compare every standard runtime-bundle version at once. The aggregator -# writes the table and JSON report before returning a failing drift status. -node scripts/ppl-lint/aggregate-versions.mjs \ +git ls-remote --tags --refs https://github.com/opensearch-project/OpenSearch.git \ + > /tmp/opensearch-release-tags.txt +node scripts/ppl-lint/plan-compatibility.mjs \ + --build-file build.gradle \ + --release-tags /tmp/opensearch-release-tags.txt \ + --compiled-version 2.19.6 \ + --sql-sha "$(git rev-parse HEAD)" \ + --osd-repository opensearch-project/OpenSearch-Dashboards \ + --osd-ref main \ + --out compatibility-plan.json + +node scripts/ppl-lint/aggregate-compatibility.mjs \ + --plan compatibility-plan.json \ --contracts integ-test/src/test/resources/ppl-lint/contracts \ - --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 --leg pr-build=legs/pr-build \ + --artifacts legs \ + --osd-sha "" \ --out drift-report.json ``` The step summary has one row per active detector. It prints the compatibility -declared by `wiring.appliesTo` next to the actual result for every engine leg. -For example, a rule with `minVersion: 3.7.0` renders `expected n/a` on 3.6 -instead of reporting drift. +declared by `wiring.appliesTo` and `grammarSurface` next to each actual result. The classifier is pure and has no cluster or OSD dependency, so its tests run anywhere: @@ -412,9 +379,8 @@ node --test "scripts/ppl-lint/__tests__/*.test.mjs" ## Discovery corpus (harvested, never enforced) -The required corpus is hand-pinned, which is what lets a mismatch red the build. -The `discovery` job builds a larger unpinned corpus to distinguish full engine -fixes from partial behavior changes. +The discovery scripts remain available as standalone investigation tooling. +They are not invoked by the multi-surface compatibility workflow. ``` harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-frontend-contract.mjs ──▶ detector report diff --git a/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs b/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs new file mode 100644 index 00000000000..13df6796325 --- /dev/null +++ b/scripts/ppl-lint/__tests__/aggregate-compatibility.test.mjs @@ -0,0 +1,500 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { resolveBackendOracle } from '../contract-schema.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'aggregate-compatibility.mjs'); +const REPOSITORY = path.resolve(HERE, '..', '..', '..'); +const CONTRACTS = path.join( + REPOSITORY, + 'integ-test', + 'src', + 'test', + 'resources', + 'ppl-lint', + 'contracts' +); +const WORKFLOW = path.join( + REPOSITORY, + '.github', + 'workflows', + 'ppl-lint-multiversion-validation.yml' +); +const RULE_IDS = JSON.parse( + fs.readFileSync(path.join(CONTRACTS, 'manifest.json'), 'utf8') +).contracts.map((file) => file.replace(/\.spec\.json$/, '')); +const CONFIGURATIONS = [ + { + id: '2.19.6-compiled', + label: '2.19.6 compiled', + engineVersion: '2.19.6', + surface: 'compiled-simplified', + executionBackend: 'standard', + engineMode: 'legacy', + artifactName: 'ppl-lint-observation-2.19.6-compiled', + exportRuntimeBundle: false, + }, + { + id: 'latest-release-runtime', + label: 'Latest release (3.8.0) runtime', + engineVersion: '3.8.0', + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-latest-release-runtime', + exportRuntimeBundle: true, + }, + { + id: 'pr-build-runtime', + label: 'PR runtime', + engineVersion: '3.8.0-SNAPSHOT', + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-pr-build-runtime', + exportRuntimeBundle: true, + }, +]; +const temporaryDirectories = []; + +function temporaryDirectory(prefix) { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + temporaryDirectories.push(directory); + return directory; +} + +after(() => { + for (const directory of temporaryDirectories) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +function version(value) { + const match = /^(\d+)\.(\d+)\.(\d+)/.exec(value); + return match.slice(1, 4).map(Number); +} + +function compare(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return 0; +} + +function applicable(spec, configuration) { + const appliesTo = spec.wiring.appliesTo || {}; + const surfaces = + spec.grammarSurface === 'both' + ? ['compiled-simplified', 'runtime-bundle'] + : [spec.grammarSurface || 'runtime-bundle']; + if (!surfaces.includes(configuration.surface)) return false; + const actual = version(configuration.engineVersion); + if (appliesTo.minVersion && compare(actual, version(appliesTo.minVersion)) < 0) return false; + if (appliesTo.maxVersion && compare(actual, version(appliesTo.maxVersion)) > 0) return false; + return !appliesTo.engine || appliesTo.engine === configuration.engineMode; +} + +function rangeMatches(range, engineVersion) { + const actual = version(engineVersion); + return String(range || '') + .trim() + .split(/\s+/) + .filter(Boolean) + .every((token) => { + const match = /^(>=|<=|>|<|=)?(.+)$/.exec(token); + const comparison = compare(actual, version(match[2])); + return ( + (!match[1] && comparison === 0) || + (match[1] === '=' && comparison === 0) || + (match[1] === '>=' && comparison >= 0) || + (match[1] === '<=' && comparison <= 0) || + (match[1] === '>' && comparison > 0) || + (match[1] === '<' && comparison < 0) + ); + }); +} + +function loadSpecs() { + return new Map( + JSON.parse(fs.readFileSync(path.join(CONTRACTS, 'manifest.json'), 'utf8')).contracts.map( + (file) => { + const spec = JSON.parse(fs.readFileSync(path.join(CONTRACTS, file), 'utf8')); + return [spec.ruleId, spec]; + } + ) + ); +} + +function selectedExpectation(spec, configuration) { + return spec.expectations.find( + (expectation) => + rangeMatches(expectation.version, configuration.engineVersion) && + (!expectation.engine || expectation.engine === configuration.engineMode) + ); +} + +function target(configuration, grammarHash) { + return { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: configuration.engineVersion, + grammarHash, + grammarBundle: + configuration.surface === 'runtime-bundle' ? 'ppl-grammar-bundle.json' : '', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }; +} + +function writeHealthyArtifacts(root) { + const specs = loadSpecs(); + for (const configuration of CONFIGURATIONS) { + const directory = path.join(root, configuration.artifactName); + fs.mkdirSync(directory, { recursive: true }); + const grammarHash = + configuration.surface === 'runtime-bundle' + ? `sha256:${configuration.id}` + : 'sha256:compiled-grammar'; + const backendTarget = target( + configuration, + configuration.surface === 'runtime-bundle' ? grammarHash : '' + ); + const detectorTarget = target(configuration, grammarHash); + const detectorResults = []; + const backendResults = []; + + for (const spec of specs.values()) { + if (!applicable(spec, configuration)) continue; + const expectation = selectedExpectation(spec, configuration); + assert.ok(expectation, `fixture expectation for ${spec.ruleId} on ${configuration.id}`); + for (const [queryName, queryDefinition] of Object.entries(spec.queries)) { + const resolved = resolveBackendOracle( + spec, + expectation.queries[queryName], + 'standard' + ); + assert.equal(resolved.status, 'applicable'); + const rejected = resolved.oracle.kind === 'rejection'; + const error = resolved.oracle.body && resolved.oracle.body.error; + detectorResults.push({ + ruleId: spec.ruleId, + queryName, + role: queryDefinition.role || 'trigger', + expected: resolved.detector.count, + actual: resolved.detector.count, + severities: + resolved.detector.count > 0 && resolved.detector.severity + ? [resolved.detector.severity] + : [], + severityMatched: true, + messageMatched: true, + executionBackend: 'standard', + }); + backendResults.push({ + ruleId: spec.ruleId, + queryName, + role: queryDefinition.role || 'trigger', + rejected, + executionBackend: 'standard', + observed: { + rejected, + httpStatus: resolved.oracle.httpStatus, + ...(error && error.type ? { type: error.type } : {}), + ...(error && error.reason ? { reason: error.reason } : {}), + }, + }); + } + } + + fs.writeFileSync(path.join(directory, 'target.json'), JSON.stringify(backendTarget)); + fs.writeFileSync( + path.join(directory, 'detector-target.json'), + JSON.stringify(detectorTarget) + ); + fs.writeFileSync( + path.join(directory, 'backend-report.json'), + JSON.stringify(backendResults) + ); + fs.writeFileSync( + path.join(directory, 'detector-report.json'), + JSON.stringify({ + schemaVersion: 2, + engineVersion: configuration.engineVersion, + grammarHash, + executionBackend: 'standard', + surface: configuration.surface, + results: detectorResults, + }) + ); + if (configuration.surface === 'runtime-bundle') { + fs.writeFileSync( + path.join(directory, 'ppl-grammar-bundle.json'), + JSON.stringify({ grammarHash }) + ); + } + fs.writeFileSync(path.join(directory, 'backend-command.txt'), 'backend command\n'); + fs.writeFileSync(path.join(directory, 'detector-command.txt'), 'detector command\n'); + } +} + +function createFixture() { + const directory = temporaryDirectory('ppl-lint-compatibility-'); + const artifacts = path.join(directory, 'legs'); + fs.mkdirSync(artifacts); + writeHealthyArtifacts(artifacts); + const plan = { + schemaVersion: 1, + sqlSha: 'candidate-sql-sha', + prTargetVersion: '3.8.0-SNAPSHOT', + normalizedPrTarget: '3.8.0', + latestEligibleGa: '3.8.0', + osd: { repository: 'example/osd', ref: 'main' }, + configurations: CONFIGURATIONS, + }; + const planFile = path.join(directory, 'compatibility-plan.json'); + fs.writeFileSync(planFile, JSON.stringify(plan)); + return { directory, artifacts, planFile }; +} + +function run(fixture) { + const reportFile = path.join(fixture.directory, 'drift-report.json'); + const summaryFile = path.join(fixture.directory, 'summary.md'); + const result = spawnSync( + process.execPath, + [ + SCRIPT, + '--plan', + fixture.planFile, + '--contracts', + CONTRACTS, + '--artifacts', + fixture.artifacts, + '--osd-sha', + 'osd-sha', + '--out', + reportFile, + '--summary', + summaryFile, + ], + { encoding: 'utf8' } + ); + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + report: fs.existsSync(reportFile) + ? JSON.parse(fs.readFileSync(reportFile, 'utf8')) + : undefined, + summary: fs.existsSync(summaryFile) ? fs.readFileSync(summaryFile, 'utf8') : '', + }; +} + +function editBackend(fixture, configurationId, ruleId, queryName, patch) { + const configuration = CONFIGURATIONS.find((entry) => entry.id === configurationId); + const file = path.join( + fixture.artifacts, + configuration.artifactName, + 'backend-report.json' + ); + const report = JSON.parse(fs.readFileSync(file, 'utf8')); + const row = report.find( + (entry) => entry.ruleId === ruleId && entry.queryName === queryName + ); + Object.assign(row, patch); + Object.assign(row.observed, patch.observed || {}); + fs.writeFileSync(file, JSON.stringify(report)); +} + +test('emits exactly 12 rules, 3 configurations, and 36 complete cells', () => { + const result = run(createFixture()); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.report.schemaVersion, 3); + assert.equal(result.report.inventory.ruleCount, 12); + assert.deepEqual(result.report.inventory.ruleIds, [...RULE_IDS].sort()); + assert.equal(result.report.configurations.length, 3); + assert.equal(result.report.matrix.length, 36); + assert.deepEqual(result.report.result, { + status: 'pass', + cellCount: 36, + compatible: 28, + notApplicable: 8, + drift: 0, + inconclusive: 0, + exitCode: 0, + }); + assert.equal( + result.summary.split('\n').filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)) + .length, + 12 + ); +}); + +test('uses surface before version when a compiled pair has multiple exclusions', () => { + const { report } = run(createFixture()); + assert.equal( + report.matrix.find( + (entry) => + entry.ruleId === 'invalid-capture-group-name' && + entry.configurationId === '2.19.6-compiled' + ).expected.reason, + 'surface' + ); + assert.equal( + report.matrix.find( + (entry) => + entry.ruleId === 'agg-on-text' && + entry.configurationId === '2.19.6-compiled' + ).expected.reason, + 'version' + ); +}); + +test('classifies a one-trigger full engine relaxation and keeps the complete table', () => { + const fixture = createFixture(); + editBackend( + fixture, + 'latest-release-runtime', + 'wildcard-source-zero-match', + 'missing-wildcard-source', + { + rejected: false, + observed: { rejected: false, httpStatus: 200, type: undefined, reason: undefined }, + } + ); + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + const cell = result.report.matrix.find( + (entry) => + entry.ruleId === 'wildcard-source-zero-match' && + entry.configurationId === 'latest-release-runtime' + ); + assert.equal(cell.classification, 'full-engine-relaxation'); + assert.deepEqual(cell.triggerSummary, { + contracted: 1, + acceptedByBackend: 1, + rejectedByBackend: 0, + missing: 0, + }); + assert.equal( + result.report.findings.find( + (entry) => + entry.ruleId === 'wildcard-source-zero-match' && + entry.configurationId === 'latest-release-runtime' + ).remediation.action, + 'scope-rule-version' + ); +}); + +test('classifies partial relaxation separately and never recommends version scoping', () => { + const fixture = createFixture(); + editBackend( + fixture, + 'latest-release-runtime', + 'union-min-datasets', + 'union-single-dataset', + { + rejected: false, + observed: { rejected: false, httpStatus: 200, type: undefined, reason: undefined }, + } + ); + const result = run(fixture); + const finding = result.report.findings.find( + (entry) => + entry.ruleId === 'union-min-datasets' && + entry.configurationId === 'latest-release-runtime' + ); + assert.equal(finding.classification, 'partial-engine-relaxation'); + assert.equal(finding.remediation.action, 'narrow-detector'); + assert.ok( + !result.report.findings.some( + (entry) => + entry.ruleId === 'union-min-datasets' && + entry.remediation.action === 'scope-rule-version' + ) + ); +}); + +test('a detector regression writes JSON and every summary row before exiting nonzero', () => { + const fixture = createFixture(); + const configuration = CONFIGURATIONS[1]; + const file = path.join( + fixture.artifacts, + configuration.artifactName, + 'detector-report.json' + ); + const report = JSON.parse(fs.readFileSync(file, 'utf8')); + report.results.find( + (entry) => + entry.ruleId === 'rex-scan-cost' && entry.queryName === 'parse-text-field' + ).actual = 0; + fs.writeFileSync(file, JSON.stringify(report)); + + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + assert.equal( + result.report.matrix.find( + (entry) => + entry.ruleId === 'rex-scan-cost' && + entry.configurationId === 'latest-release-runtime' + ).classification, + 'detector-regression' + ); + assert.equal( + result.summary + .split('### Blocking findings')[0] + .split('\n') + .filter((line) => /^\| `[a-z0-9-]+` \|/.test(line)).length, + 12 + ); + assert.match(result.stderr, /after writing the complete report/); +}); + +test('a missing observation preserves the full column as inconclusive', () => { + const fixture = createFixture(); + fs.rmSync( + path.join( + fixture.artifacts, + CONFIGURATIONS[1].artifactName + ), + { recursive: true } + ); + const result = run(fixture); + assert.equal(result.status, 1); + assert.equal(result.report.matrix.length, 36); + const column = result.report.matrix.filter( + (entry) => entry.configurationId === 'latest-release-runtime' + ); + assert.equal(column.length, 12); + assert.ok(column.every((entry) => entry.status === 'inconclusive')); + assert.equal(result.report.result.inconclusive, 12); +}); + +test('workflow uploads the mandatory report before the only enforcement step', () => { + const workflow = fs.readFileSync(WORKFLOW, 'utf8'); + const upload = workflow.indexOf('- name: Upload drift report'); + const evidence = workflow.indexOf('- name: Upload compatibility evidence'); + const enforce = workflow.indexOf('- name: Fail after publishing compatibility results'); + assert.ok(upload > 0 && evidence > upload && enforce > evidence); + const reportStep = workflow.slice(upload, evidence); + assert.match(reportStep, /path: drift-report\.json/); + assert.match(reportStep, /if-no-files-found: error/); + assert.equal( + (workflow.match(/- name: Fail after publishing compatibility results/g) || []) + .length, + 1 + ); +}); diff --git a/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs b/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs new file mode 100644 index 00000000000..78e6dc45a68 --- /dev/null +++ b/scripts/ppl-lint/__tests__/plan-compatibility.test.mjs @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; + +import { + createPlan, + parseVersion, + releaseVersions, + selectLatestGaAtOrBelow, +} from '../plan-compatibility.mjs'; + +const temporaryDirectories = []; + +function temporaryDirectory() { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-plan-')); + temporaryDirectories.push(directory); + return directory; +} + +after(() => { + for (const directory of temporaryDirectories) { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); + +test('normalizes prerelease and build suffixes', () => { + assert.deepEqual(parseVersion('3.8.0-SNAPSHOT'), { + normalized: '3.8.0', + parts: [3, 8, 0], + }); + assert.deepEqual(parseVersion('3.8.0+build.42'), { + normalized: '3.8.0', + parts: [3, 8, 0], + }); +}); + +test('only exact semantic version tags count as official GA candidates', () => { + const tags = [ + 'a refs/tags/3.7.0', + 'b refs/tags/3.8.0-alpha1', + 'c refs/tags/3.7.1', + 'd refs/tags/v3.8.0', + 'e refs/tags/3.8.0', + ].join('\n'); + assert.deepEqual( + releaseVersions(tags).map((entry) => entry.version), + ['3.7.0', '3.7.1', '3.8.0'] + ); +}); + +test('selects the highest GA at or below the normalized PR target', () => { + const tags = ['3.6.0', '3.7.0', '3.7.2', '3.8.0', '3.9.0'].join('\n'); + assert.equal( + selectLatestGaAtOrBelow(tags, parseVersion('3.8.0-SNAPSHOT')), + '3.8.0' + ); + assert.equal( + selectLatestGaAtOrBelow(tags, parseVersion('3.7.5-SNAPSHOT')), + '3.7.2' + ); +}); + +test('plans one compiled and two runtime configurations', () => { + const directory = temporaryDirectory(); + const buildFile = path.join(directory, 'build.gradle'); + fs.writeFileSync( + buildFile, + 'opensearch_version = System.getProperty("opensearch.version", "3.8.0-SNAPSHOT")\n' + ); + const plan = createPlan({ + buildFile, + releaseTags: ['a refs/tags/3.7.0', 'b refs/tags/3.8.0'].join('\n'), + compiledVersion: '2.19.6', + sqlSha: 'sql-sha', + osdRepository: 'example/OpenSearch-Dashboards', + osdRef: 'feature', + }); + + assert.equal(plan.latestEligibleGa, '3.8.0'); + assert.deepEqual( + plan.configurations.map((configuration) => [ + configuration.id, + configuration.surface, + configuration.engineVersion, + ]), + [ + ['2.19.6-compiled', 'compiled-simplified', '2.19.6'], + ['latest-release-runtime', 'runtime-bundle', '3.8.0'], + ['pr-build-runtime', 'runtime-bundle', '3.8.0-SNAPSHOT'], + ] + ); + assert.equal(plan.releasedTargets.include.length, 2); +}); diff --git a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs index e8ffa5ad281..42248ce3adc 100644 --- a/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs +++ b/scripts/ppl-lint/__tests__/run-frontend-contract.test.mjs @@ -15,6 +15,7 @@ import { assertActiveShippingContracts, buildCensus, buildFrontendExecutionError, + compatibilityExclusion, evaluateFrontendAssertions, selectManifestContractNames, } from '../run-frontend-contract.mjs'; @@ -28,6 +29,54 @@ const RANGE = { endColumn: 14, }; +test('compatibility exclusion uses surface, version, then engine precedence', () => { + const runtimeCalciteRule = { + grammarSurface: 'runtime-bundle', + wiring: { + appliesTo: { minVersion: '3.4.0', engine: 'calcite' }, + }, + }; + assert.equal( + compatibilityExclusion( + runtimeCalciteRule, + '2.19.6', + 'compiled-simplified', + 'legacy' + ).reason, + 'surface' + ); + assert.equal( + compatibilityExclusion( + { ...runtimeCalciteRule, grammarSurface: 'both' }, + '2.19.6', + 'compiled-simplified', + 'legacy' + ).reason, + 'version' + ); + assert.equal( + compatibilityExclusion( + { + grammarSurface: 'both', + wiring: { appliesTo: { engine: 'calcite' } }, + }, + '3.8.0', + 'runtime-bundle', + 'legacy' + ).reason, + 'engine' + ); + assert.equal( + compatibilityExclusion( + runtimeCalciteRule, + '3.8.0-SNAPSHOT', + 'runtime-bundle', + 'calcite' + ), + undefined + ); +}); + test('exact lint assertions materialize the effective deterministic edit', () => { const result = evaluateFrontendAssertions({ channel: 'lint', diff --git a/scripts/ppl-lint/aggregate-compatibility.mjs b/scripts/ppl-lint/aggregate-compatibility.mjs new file mode 100644 index 00000000000..e46a872d93f --- /dev/null +++ b/scripts/ppl-lint/aggregate-compatibility.mjs @@ -0,0 +1,961 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { + assertContractSchema, + assertExactQueryCoverage, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; + +const ACTIVE_RULE_IDS = [ + 'agg-on-text', + 'division-by-zero', + 'enabled-false-object', + 'field-validation', + 'invalid-capture-group-name', + 'multisearch-min-subsearch', + 'replace-wildcard-asymmetry', + 'rex-scan-cost', + 'type-mismatch-numeric', + 'union-min-datasets', + 'unsupported-window-function-in-eventstats', + 'wildcard-source-zero-match', +]; + +function fatal(message) { + process.stderr.write(`[ppl-lint-compatibility] FATAL: ${message}\n`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { out: 'drift-report.json', summary: '', osdSha: '' }; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + const value = argv[++index]; + if (value === undefined) fatal(`${key} requires a value`); + if (key === '--plan') args.plan = value; + else if (key === '--contracts') args.contracts = value; + else if (key === '--artifacts') args.artifacts = value; + else if (key === '--osd-sha') args.osdSha = value; + else if (key === '--out') args.out = value; + else if (key === '--summary') args.summary = value; + else fatal(`unknown argument ${JSON.stringify(key)}`); + } + for (const field of ['plan', 'contracts', 'artifacts']) { + if (!args[field]) fatal(`--${field} is required`); + } + return args; +} + +function readRequiredJson(file) { + if (!fs.existsSync(file)) fatal(`required JSON file not found: ${file}`); + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + fatal(`could not parse ${file}: ${error.message}`); + } +} + +function readOptionalJson(file) { + if (!fs.existsSync(file)) { + return { value: undefined, error: `missing ${path.basename(file)}` }; + } + try { + return { value: JSON.parse(fs.readFileSync(file, 'utf8')), error: undefined }; + } catch (error) { + return { value: undefined, error: `invalid ${path.basename(file)}: ${error.message}` }; + } +} + +function readOptionalText(file) { + try { + return fs.readFileSync(file, 'utf8').trim(); + } catch { + return ''; + } +} + +function parseVersion(value) { + const match = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(value || '')); + return match ? match.slice(1, 4).map((part) => Number(part || 0)) : undefined; +} + +function compareVersion(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] < right[index] ? -1 : 1; + } + return 0; +} + +function inVersionRange(version, minVersion, maxVersion) { + const actual = parseVersion(version); + if (!actual) return false; + const min = parseVersion(minVersion); + const max = parseVersion(maxVersion); + if (min && compareVersion(actual, min) < 0) return false; + if (max && compareVersion(actual, max) > 0) return false; + return true; +} + +function matchesExpectationRange(range, version) { + if (!range || !String(range).trim()) return true; + const actual = parseVersion(version); + if (!actual) return false; + for (const token of String(range).trim().split(/\s+/)) { + const match = /^(>=|<=|>|<|=)?(\d+(?:\.\d+){0,2})$/.exec(token); + if (!match) return false; + const expected = parseVersion(match[2]); + const comparison = compareVersion(actual, expected); + const operator = match[1] || '='; + if ( + !( + (operator === '>=' && comparison >= 0) || + (operator === '<=' && comparison <= 0) || + (operator === '>' && comparison > 0) || + (operator === '<' && comparison < 0) || + (operator === '=' && comparison === 0) + ) + ) { + return false; + } + } + return true; +} + +function validatePlan(plan) { + if (!plan || plan.schemaVersion !== 1 || !Array.isArray(plan.configurations)) { + fatal('compatibility plan must be schemaVersion 1 with a configurations array'); + } + if (plan.configurations.length !== 3) { + fatal(`compatibility plan must contain exactly 3 configurations, found ${plan.configurations.length}`); + } + const ids = new Set(); + for (const configuration of plan.configurations) { + for (const field of [ + 'id', + 'label', + 'engineVersion', + 'surface', + 'executionBackend', + 'engineMode', + 'artifactName', + ]) { + if (typeof configuration[field] !== 'string' || !configuration[field]) { + fatal(`plan configuration ${JSON.stringify(configuration.id)} has invalid ${field}`); + } + } + if (!['compiled-simplified', 'runtime-bundle'].includes(configuration.surface)) { + fatal(`plan configuration ${configuration.id} has unknown surface ${configuration.surface}`); + } + if (!['calcite', 'legacy'].includes(configuration.engineMode)) { + fatal( + `plan configuration ${configuration.id} has unknown engine mode ` + + configuration.engineMode + ); + } + if (ids.has(configuration.id)) fatal(`duplicate plan configuration ${configuration.id}`); + ids.add(configuration.id); + } +} + +function loadContracts(dir) { + const manifestPath = path.join(dir, 'manifest.json'); + const manifest = readRequiredJson(manifestPath); + if (!Array.isArray(manifest.contracts)) fatal(`${manifestPath} contracts must be an array`); + const contracts = new Map(); + for (const file of manifest.contracts) { + const specPath = path.join(dir, file); + const spec = readRequiredJson(specPath); + try { + assertContractSchema(spec); + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new Error('expectations must be a non-empty array'); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + } + } catch (error) { + fatal(`invalid ${specPath}: ${error.message}`); + } + if (contracts.has(spec.ruleId)) fatal(`duplicate active rule id ${spec.ruleId}`); + contracts.set(spec.ruleId, { file, spec }); + } + const actual = [...contracts.keys()].sort(); + if (JSON.stringify(actual) !== JSON.stringify(ACTIVE_RULE_IDS)) { + fatal( + `active manifest must contain exactly the approved 12 rules; expected ` + + `${JSON.stringify(ACTIVE_RULE_IDS)}, got ${JSON.stringify(actual)}` + ); + } + return contracts; +} + +function expectedScope(spec, configuration) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const declaredSurface = spec.grammarSurface || 'runtime-bundle'; + const surfaces = + declaredSurface === 'both' + ? ['compiled-simplified', 'runtime-bundle'] + : [declaredSurface]; + const expected = { + applicable: true, + engine: appliesTo.engine || 'any', + minVersion: appliesTo.minVersion || null, + maxVersion: appliesTo.maxVersion || null, + surfaces, + }; + + if (!surfaces.includes(configuration.surface)) { + return { ...expected, applicable: false, reason: 'surface' }; + } + if ( + !inVersionRange( + configuration.engineVersion, + appliesTo.minVersion, + appliesTo.maxVersion + ) + ) { + return { ...expected, applicable: false, reason: 'version' }; + } + if (appliesTo.engine && appliesTo.engine !== configuration.engineMode) { + return { ...expected, applicable: false, reason: 'engine' }; + } + return expected; +} + +function selectExpectation(spec, configuration) { + const matches = spec.expectations.filter( + (expectation) => + matchesExpectationRange(expectation.version, configuration.engineVersion) && + (!expectation.engine || expectation.engine === configuration.engineMode) + ); + return matches.length === 1 ? matches[0] : undefined; +} + +function indexDetectorReport(report) { + if (!report || typeof report !== 'object' || !Array.isArray(report.results)) { + throw new Error('detector-report.json must contain a results array'); + } + const rows = new Map(); + for (const row of report.results) { + if (!row || typeof row.ruleId !== 'string' || typeof row.queryName !== 'string') { + throw new Error('detector report rows require ruleId and queryName'); + } + const key = `${row.ruleId}::${row.queryName}`; + if (rows.has(key)) throw new Error(`duplicate detector row ${key}`); + rows.set(key, row); + } + return rows; +} + +function loadEvidence(configuration, artifactsRoot, sqlSha) { + const dir = path.join(artifactsRoot, configuration.artifactName); + const errors = []; + const backendTargetRead = readOptionalJson(path.join(dir, 'target.json')); + const detectorTargetRead = readOptionalJson(path.join(dir, 'detector-target.json')); + const detectorRead = readOptionalJson(path.join(dir, 'detector-report.json')); + const backendRead = readOptionalJson(path.join(dir, 'backend-report.json')); + const bundleRead = + configuration.surface === 'runtime-bundle' + ? readOptionalJson(path.join(dir, 'ppl-grammar-bundle.json')) + : { value: undefined, error: undefined }; + + let backendTarget; + let detectorTarget; + let detectorRows = new Map(); + let backendRows = new Map(); + + for (const item of [backendTargetRead, detectorTargetRead, detectorRead, backendRead, bundleRead]) { + if (item.error) errors.push(item.error); + } + if (backendTargetRead.value) { + try { + backendTarget = normalizeTarget(backendTargetRead.value); + } catch (error) { + errors.push(`invalid target.json: ${error.message}`); + } + } + if (detectorTargetRead.value) { + try { + detectorTarget = normalizeTarget(detectorTargetRead.value); + } catch (error) { + errors.push(`invalid detector-target.json: ${error.message}`); + } + } + if (detectorRead.value) { + try { + detectorRows = indexDetectorReport(detectorRead.value); + } catch (error) { + errors.push(error.message); + } + } + if (backendRead.value && backendTarget) { + try { + backendRows = indexBackendReport(backendRead.value, backendTarget); + } catch (error) { + errors.push(`invalid backend-report.json: ${error.message}`); + } + } + + if (backendTarget) { + if ( + !backendTarget.engineVersion.startsWith( + configuration.engineVersion.replace(/[-+].*$/, '') + ) + ) { + errors.push( + `target engine ${backendTarget.engineVersion} does not match planned ` + + configuration.engineVersion + ); + } + if (backendTarget.executionBackend !== configuration.executionBackend) { + errors.push( + `target backend ${backendTarget.executionBackend} does not match planned ` + + configuration.executionBackend + ); + } + if (sqlSha && backendTarget.sqlSha && backendTarget.sqlSha !== sqlSha) { + errors.push(`target SQL SHA ${backendTarget.sqlSha} does not match planned ${sqlSha}`); + } + } + if (detectorTarget && detectorRead.value) { + for (const field of ['engineVersion', 'grammarHash', 'executionBackend']) { + if (detectorRead.value[field] !== detectorTarget[field]) { + errors.push( + `detector report ${field} ${JSON.stringify(detectorRead.value[field])} does not match ` + + `detector target ${JSON.stringify(detectorTarget[field])}` + ); + } + } + if (detectorRead.value.surface !== configuration.surface) { + errors.push( + `detector surface ${JSON.stringify(detectorRead.value.surface)} does not match planned ` + + configuration.surface + ); + } + } + if ( + bundleRead.value && + detectorTarget && + bundleRead.value.grammarHash !== detectorTarget.grammarHash + ) { + errors.push('runtime grammar bundle hash does not match detector target'); + } + + return { + dir, + errors: [...new Set(errors)], + backendTarget, + detectorTarget, + detectorReport: detectorRead.value, + detectorRows, + backendRows, + backendCommand: readOptionalText(path.join(dir, 'backend-command.txt')), + detectorCommand: readOptionalText(path.join(dir, 'detector-command.txt')), + }; +} + +function detectorActual(row, evidence) { + if (evidence.errors.length > 0 && !evidence.detectorReport) { + return { outcome: 'error', error: evidence.errors.join('; ') }; + } + if (!row) return { outcome: 'missing' }; + if (row.outcome === 'error') { + return { outcome: 'error', error: row.error || 'frontend execution failed' }; + } + if (row.outcome === 'not-applicable' || row.notApplicable) { + return { outcome: 'error', error: row.notApplicable || 'unexpected not-applicable row' }; + } + return { + outcome: 'observed', + count: row.actual, + severities: row.severities || [], + diagnostics: row.diagnostics || [], + }; +} + +function backendActual(row, evidence) { + if (evidence.errors.length > 0 && evidence.backendRows.size === 0) { + return { outcome: 'error', error: evidence.errors.join('; ') }; + } + if (!row) return { outcome: 'missing' }; + const state = classifyBackendReportRow(row); + if (state.status !== 'observed') { + return { + outcome: state.status === 'error' ? 'error' : 'error', + error: row.error || `backend outcome was ${state.status}`, + }; + } + const observed = row.observed || {}; + return { + outcome: 'observed', + rejected: state.rejected, + httpStatus: observed.httpStatus, + errorType: observed.type, + errorReason: observed.reason, + }; +} + +function expectedCase(spec, queryExpectation) { + const resolved = resolveBackendOracle(spec, queryExpectation, 'standard'); + if (resolved.status !== 'applicable') { + throw new Error(resolved.reason || 'standard backend oracle is not applicable'); + } + const expectedBackend = { + kind: resolved.oracle.kind, + httpStatus: resolved.oracle.httpStatus, + }; + const error = resolved.oracle.body && resolved.oracle.body.error; + if (error && error.type) expectedBackend.errorType = error.type; + if (error && error.reason) expectedBackend.errorReason = error.reason; + return { + detector: { + count: resolved.detector.count, + ...(resolved.detector.severity ? { severity: resolved.detector.severity } : {}), + ...(resolved.detector.matchMessage + ? { message: resolved.detector.matchMessage } + : {}), + ...(resolved.detector.messageEquals + ? { message: resolved.detector.messageEquals } + : {}), + }, + backend: expectedBackend, + frontend: resolved.frontend, + }; +} + +function compareDetector(expected, actual, row) { + if (actual.outcome !== 'observed') return []; + const differences = []; + if (actual.count !== expected.count) differences.push('detector.count'); + if (expected.severity && row.severityMatched === false) differences.push('detector.severity'); + if (expected.message && row.messageMatched === false) differences.push('detector.message'); + for (const field of [ + 'deterministicFixMatched', + 'fixMatched', + 'rawMessageMatched', + 'totalErrorsMatched', + ]) { + if (row[field] === false) differences.push(`detector.${field}`); + } + for (const [field, matched] of Object.entries(row.assertions || {})) { + if (matched === false && !['count', 'severity'].includes(field)) { + differences.push(`detector.${field}`); + } + } + return [...new Set(differences)]; +} + +function compareBackend(expected, actual, row) { + if (actual.outcome !== 'observed') return []; + const differences = []; + const expectedRejected = expected.kind === 'rejection'; + if (actual.rejected !== expectedRejected) differences.push('backend.rejected'); + if ( + Number.isInteger(expected.httpStatus) && + actual.httpStatus !== expected.httpStatus + ) { + differences.push('backend.httpStatus'); + } + if (expected.errorType && actual.errorType !== expected.errorType) { + differences.push('backend.errorType'); + } + if (expected.errorReason && actual.errorReason !== expected.errorReason) { + differences.push('backend.errorReason'); + } + if (row.outcome === 'observed-mismatch' || row.error) { + differences.push('backend.result'); + } + return [...new Set(differences)]; +} + +function aggregateActual(cases, side) { + const actuals = cases.map((entry) => entry.actual[side]); + const errored = actuals.find((actual) => actual.outcome === 'error'); + if (errored) return { outcome: 'error', error: errored.error }; + if (actuals.some((actual) => actual.outcome === 'missing')) return { outcome: 'missing' }; + if (side === 'detector') { + return { + outcome: 'observed', + diagnosticCount: actuals.reduce((sum, actual) => sum + (actual.count || 0), 0), + }; + } + const rejected = actuals.filter((actual) => actual.rejected === true).length; + return { outcome: 'observed', rejected, observedCases: actuals.length }; +} + +function reasonForIncomplete(cases, evidence) { + for (const entry of cases) { + if (entry.actual.detector.outcome !== 'observed') { + return { + code: + entry.actual.detector.outcome === 'missing' + ? 'missing-detector-row' + : 'detector-error', + message: + entry.actual.detector.error || + `No detector result was produced for ${entry.ruleId}::${entry.queryName}.`, + }; + } + if (entry.actual.backend.outcome !== 'observed') { + return { + code: + entry.actual.backend.outcome === 'missing' + ? 'missing-backend-row' + : 'backend-error', + message: + entry.actual.backend.error || + `No backend result was produced for ${entry.ruleId}::${entry.queryName}.`, + }; + } + } + return { + code: 'invalid-leg-identity', + message: evidence.errors.join('; ') || 'The configuration did not produce trustworthy evidence.', + }; +} + +function classifyCell(cases) { + const triggerCases = cases.filter((entry) => entry.role === 'trigger'); + const rejectionTriggers = triggerCases.filter( + (entry) => entry.expected.backend.kind === 'rejection' + ); + const accepted = rejectionTriggers.filter( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.actual.backend.rejected === false + ); + const rejected = rejectionTriggers.filter( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.actual.backend.rejected === true + ); + const controls = cases.filter((entry) => entry.role !== 'trigger'); + const controlsProveSupport = controls.every( + (entry) => + entry.actual.backend.outcome === 'observed' && + entry.differences.every((difference) => !difference.startsWith('backend.')) + ); + const detectorDifferences = cases.flatMap((entry) => + entry.differences.filter((difference) => difference.startsWith('detector.')) + ); + const backendDifferences = cases.flatMap((entry) => + entry.differences.filter((difference) => difference.startsWith('backend.')) + ); + + const triggerSummary = { + contracted: rejectionTriggers.length, + acceptedByBackend: accepted.length, + rejectedByBackend: rejected.length, + missing: rejectionTriggers.filter( + (entry) => entry.actual.backend.outcome !== 'observed' + ).length, + }; + if ( + rejectionTriggers.length > 0 && + accepted.length === rejectionTriggers.length && + controlsProveSupport + ) { + return { classification: 'full-engine-relaxation', triggerSummary }; + } + if (accepted.length > 0 && rejected.length > 0) { + return { classification: 'partial-engine-relaxation', triggerSummary }; + } + if (backendDifferences.length === 0 && detectorDifferences.length > 0) { + return { classification: 'detector-regression', triggerSummary }; + } + if (backendDifferences.length > 0) { + return { classification: 'contract-drift', triggerSummary }; + } + return { classification: undefined, triggerSummary }; +} + +function remediation(classification) { + if (classification === 'detector-regression') { + return { + action: 'update-detector', + scope: 'detector-only', + detail: 'Backend behavior is unchanged; keep appliesTo unchanged.', + }; + } + if (classification === 'full-engine-relaxation') { + return { + action: 'scope-rule-version', + scope: 'appliesTo', + detail: + 'Every contracted trigger is accepted and controls remain supported; stop applying ' + + 'this rule to this version range.', + }; + } + if (classification === 'partial-engine-relaxation') { + return { + action: 'narrow-detector', + scope: 'detector-only', + detail: 'Keep the rule active for this version and narrow it to the forms the backend still rejects.', + }; + } + return { + action: 'update-contract', + scope: 'oracle', + detail: 'Confirm the backend behavior change is intentional before updating the pinned contract.', + }; +} + +function expectedDescription(spec) { + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const parts = []; + if (appliesTo.engine) { + parts.push(appliesTo.engine === 'calcite' ? 'Calcite' : appliesTo.engine); + } + if (appliesTo.minVersion && appliesTo.maxVersion) { + parts.push(`>= ${appliesTo.minVersion}, <= ${appliesTo.maxVersion}`); + } else if (appliesTo.minVersion) { + parts.push(`>= ${appliesTo.minVersion}`); + } else if (appliesTo.maxVersion) { + parts.push(`<= ${appliesTo.maxVersion}`); + } else { + parts.push('all versions'); + } + parts.push((spec.grammarSurface || 'runtime-bundle') === 'both' ? 'both' : 'runtime only'); + return parts.join('; '); +} + +function markdownCell(row) { + if (row.status === 'n/a') return `n/a (${row.expected.reason})`; + if (row.status === 'drift') return '**drift**'; + if (row.status === 'inconclusive') return '**inconclusive**'; + return 'compatible'; +} + +function renderMarkdown(report, contracts) { + const lines = [ + `## PPL lint compatibility: ${report.result.status.toUpperCase()}`, + '', + `SQL: \`${report.candidate.sqlSha.slice(0, 9)}\` `, + `OSD: \`${report.candidate.osd.repository} @ ${report.candidate.osd.sha || report.candidate.osd.ref}\` `, + `Rules: ${report.inventory.ruleCount} `, + `Configurations: ${report.configurations.length} `, + `Blocking results: ${report.result.drift} drift, ${report.result.inconclusive} inconclusive`, + '', + `| Rule | Expected compatibility | ${report.configurations + .map((configuration) => configuration.label) + .join(' | ')} |`, + `| --- | --- | ${report.configurations.map(() => '---').join(' | ')} |`, + ]; + for (const ruleId of ACTIVE_RULE_IDS) { + const spec = contracts.get(ruleId).spec; + const cells = report.configurations.map((configuration) => + markdownCell( + report.matrix.find( + (entry) => + entry.ruleId === ruleId && entry.configurationId === configuration.id + ) + ) + ); + lines.push( + `| \`${ruleId}\` | ${expectedDescription(spec)} | ${cells.join(' | ')} |` + ); + } + lines.push(''); + + const blocking = report.findings.filter((finding) => finding.blocking); + if (blocking.length > 0) { + lines.push('### Blocking findings', ''); + lines.push('| Rule | Configuration | Classification | Evidence | Action |'); + lines.push('| --- | --- | --- | --- | --- |'); + for (const finding of blocking) { + const evidence = + finding.reason?.message || + `${finding.evidence.triggerSummary.acceptedByBackend}/` + + `${finding.evidence.triggerSummary.contracted} contracted triggers accepted`; + lines.push( + `| \`${finding.ruleId}\` | ${finding.configurationLabel} | ` + + `${finding.classification || 'inconclusive'} | ${evidence.replace(/\|/g, '\\|')} | ` + + `${finding.remediation.detail.replace(/\|/g, '\\|')} |` + ); + } + lines.push(''); + } + lines.push('### Published evidence', ''); + lines.push('| Output | Location |'); + lines.push('| --- | --- |'); + lines.push('| Full table | `Aggregate rule compatibility` step summary |'); + lines.push('| Machine-readable report | `ppl-lint-multiversion-drift/drift-report.json` |'); + lines.push('| Detector logs and target identities | `ppl-lint-multiversion-evidence` |'); + return lines.join('\n'); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const plan = readRequiredJson(args.plan); + validatePlan(plan); + const contracts = loadContracts(args.contracts); + const evidenceByConfiguration = new Map( + plan.configurations.map((configuration) => [ + configuration.id, + loadEvidence(configuration, args.artifacts, plan.sqlSha), + ]) + ); + const configurations = plan.configurations.map((configuration) => { + const evidence = evidenceByConfiguration.get(configuration.id); + return { + id: configuration.id, + label: configuration.label, + engineVersion: + (evidence.backendTarget && evidence.backendTarget.engineVersion) || + configuration.engineVersion, + surface: configuration.surface, + executionBackend: configuration.executionBackend, + engineMode: configuration.engineMode, + grammar: { + source: + configuration.surface === 'runtime-bundle' + ? 'engine-runtime-bundle' + : 'osd-compiled', + hash: (evidence.detectorTarget && evidence.detectorTarget.grammarHash) || null, + }, + }; + }); + + const matrix = []; + const cases = []; + const findings = []; + + for (const ruleId of ACTIVE_RULE_IDS) { + const { spec } = contracts.get(ruleId); + for (const configuration of plan.configurations) { + const expected = expectedScope(spec, configuration); + if (!expected.applicable) { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'n/a', + expected, + actual: null, + }); + continue; + } + + const evidence = evidenceByConfiguration.get(configuration.id); + const expectation = selectExpectation(spec, configuration); + if (!expectation) { + const reason = { + code: 'missing-contract-expectation', + message: + `No unique expectation covers ${ruleId} on ${configuration.engineVersion} ` + + `(${configuration.engineMode}).`, + }; + const cell = { + ruleId, + configurationId: configuration.id, + status: 'inconclusive', + expected, + actual: { + detector: { outcome: 'missing' }, + backend: { outcome: 'missing' }, + }, + reason, + }; + matrix.push(cell); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + blocking: true, + reason, + remediation: { + action: 'fix-test-leg', + scope: 'contract', + detail: 'Add or correct the reviewed expectation, then rerun compatibility validation.', + }, + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + continue; + } + + const cellCases = []; + for (const [queryName, queryDefinition] of Object.entries(spec.queries || {})) { + const key = `${ruleId}::${queryName}`; + const detectorRow = evidence.detectorRows.get(key); + const backendRow = evidence.backendRows.get(key); + let expectedEvidence; + try { + expectedEvidence = expectedCase(spec, expectation.queries[queryName]); + } catch (error) { + fatal(`invalid ${ruleId}::${queryName} expectation: ${error.message}`); + } + const actual = { + detector: detectorActual(detectorRow, evidence), + backend: backendActual(backendRow, evidence), + }; + const differences = [ + ...compareDetector(expectedEvidence.detector, actual.detector, detectorRow || {}), + ...compareBackend(expectedEvidence.backend, actual.backend, backendRow || {}), + ]; + const entry = { + key: `${configuration.id}::${ruleId}::${queryName}`, + ruleId, + configurationId: configuration.id, + queryName, + role: queryDefinition.role || 'trigger', + query: String(queryDefinition.query || '').split('{{index}}').join(spec.index), + expected: { + detector: expectedEvidence.detector, + backend: expectedEvidence.backend, + }, + actual, + differences, + }; + cases.push(entry); + cellCases.push(entry); + } + + const actual = { + detector: aggregateActual(cellCases, 'detector'), + backend: aggregateActual(cellCases, 'backend'), + }; + const incomplete = + evidence.errors.length > 0 || + cellCases.some( + (entry) => + entry.actual.detector.outcome !== 'observed' || + entry.actual.backend.outcome !== 'observed' + ); + if (incomplete) { + const reason = reasonForIncomplete(cellCases, evidence); + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'inconclusive', + expected, + actual, + reason, + caseKeys: cellCases.map((entry) => entry.key), + }); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + blocking: true, + reason, + evidence: { caseKeys: cellCases.map((entry) => entry.key) }, + remediation: { + action: 'fix-test-leg', + scope: 'test-leg', + detail: 'Fix or rerun this test leg before recommending a product change.', + }, + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + continue; + } + + const classification = classifyCell(cellCases); + if (classification.classification) { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'drift', + expected, + actual, + classification: classification.classification, + triggerSummary: classification.triggerSummary, + caseKeys: cellCases.map((entry) => entry.key), + }); + findings.push({ + ruleId, + configurationId: configuration.id, + configurationLabel: configuration.label, + classification: classification.classification, + blocking: true, + evidence: { + caseKeys: cellCases.map((entry) => entry.key), + triggerSummary: classification.triggerSummary, + }, + remediation: remediation(classification.classification), + reproduction: { + detectorCommand: evidence.detectorCommand, + backendCommand: evidence.backendCommand, + }, + }); + } else { + matrix.push({ + ruleId, + configurationId: configuration.id, + status: 'compatible', + expected, + actual, + caseKeys: cellCases.map((entry) => entry.key), + }); + } + } + } + + const compatible = matrix.filter((entry) => entry.status === 'compatible').length; + const notApplicable = matrix.filter((entry) => entry.status === 'n/a').length; + const drift = matrix.filter((entry) => entry.status === 'drift').length; + const inconclusive = matrix.filter((entry) => entry.status === 'inconclusive').length; + const cellCount = matrix.length; + if ( + cellCount !== ACTIVE_RULE_IDS.length * configurations.length || + compatible + notApplicable + drift + inconclusive !== cellCount + ) { + fatal('internal matrix accounting invariant failed'); + } + + const report = { + schemaVersion: 3, + candidate: { + sqlSha: plan.sqlSha, + osd: { + repository: plan.osd.repository, + ref: plan.osd.ref, + sha: args.osdSha, + }, + }, + inventory: { + ruleCount: ACTIVE_RULE_IDS.length, + ruleIds: ACTIVE_RULE_IDS, + }, + configurations, + matrix, + cases, + findings, + result: { + status: drift + inconclusive === 0 ? 'pass' : 'fail', + cellCount, + compatible, + notApplicable, + drift, + inconclusive, + exitCode: drift + inconclusive === 0 ? 0 : 1, + }, + }; + + fs.writeFileSync(args.out, `${JSON.stringify(report, null, 2)}\n`); + const markdown = renderMarkdown(report, contracts); + process.stdout.write(`${markdown}\n`); + if (args.summary) fs.appendFileSync(args.summary, `${markdown}\n`); + if (report.result.exitCode !== 0) { + process.stderr.write( + `Rule compatibility validation failed after writing the complete report: ` + + `${drift} drift, ${inconclusive} inconclusive.\n` + ); + process.exitCode = report.result.exitCode; + } +} + +main(); diff --git a/scripts/ppl-lint/plan-compatibility.mjs b/scripts/ppl-lint/plan-compatibility.mjs new file mode 100644 index 00000000000..bdd77e06581 --- /dev/null +++ b/scripts/ppl-lint/plan-compatibility.mjs @@ -0,0 +1,187 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +function fail(message) { + throw new Error(message); +} + +export function parseVersion(value) { + const match = /^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$/.exec(String(value || '').trim()); + if (!match) return undefined; + return { + normalized: `${match[1]}.${match[2]}.${match[3]}`, + parts: match.slice(1, 4).map(Number), + }; +} + +function compareVersions(left, right) { + for (let index = 0; index < 3; index += 1) { + if (left[index] !== right[index]) return left[index] - right[index]; + } + return 0; +} + +export function readPrTarget(buildFile) { + const source = fs.readFileSync(buildFile, 'utf8'); + const match = + /opensearch_version\s*=\s*System\.getProperty\(\s*["']opensearch\.version["']\s*,\s*["']([^"']+)["']\s*\)/.exec( + source + ); + if (!match) { + fail(`could not resolve the default opensearch.version from ${buildFile}`); + } + const parsed = parseVersion(match[1]); + if (!parsed) { + fail(`default opensearch.version ${JSON.stringify(match[1])} is not semantic version X.Y.Z`); + } + return { raw: match[1], normalized: parsed.normalized, parts: parsed.parts }; +} + +export function releaseVersions(text) { + const versions = new Map(); + for (const line of String(text || '').split(/\r?\n/)) { + const refMatch = /refs\/tags\/([^\s^]+)$/.exec(line.trim()); + const candidate = refMatch ? refMatch[1] : line.trim(); + if (!/^\d+\.\d+\.\d+$/.test(candidate)) continue; + const parsed = parseVersion(candidate); + versions.set(parsed.normalized, parsed.parts); + } + return [...versions.entries()] + .map(([version, parts]) => ({ version, parts })) + .sort((left, right) => compareVersions(left.parts, right.parts)); +} + +export function selectLatestGaAtOrBelow(tags, target) { + const eligible = releaseVersions(tags).filter( + (release) => compareVersions(release.parts, target.parts) <= 0 + ); + if (eligible.length === 0) { + fail(`no official GA release tag exists at or below ${target.normalized}`); + } + return eligible.at(-1).version; +} + +function parseArgs(argv) { + const args = {}; + for (let index = 0; index < argv.length; index += 1) { + const key = argv[index]; + if (!key.startsWith('--')) fail(`unexpected argument ${JSON.stringify(key)}`); + const value = argv[++index]; + if (value === undefined) fail(`${key} requires a value`); + args[key.slice(2)] = value; + } + for (const required of [ + 'build-file', + 'release-tags', + 'compiled-version', + 'sql-sha', + 'osd-repository', + 'osd-ref', + 'out', + ]) { + if (!args[required]) fail(`--${required} is required`); + } + return args; +} + +export function createPlan({ + buildFile, + releaseTags, + compiledVersion, + sqlSha, + osdRepository, + osdRef, +}) { + const target = readPrTarget(buildFile); + const compiled = parseVersion(compiledVersion); + if (!compiled || compiled.normalized !== compiledVersion) { + fail(`compiled version ${JSON.stringify(compiledVersion)} must be exact semantic version X.Y.Z`); + } + const latestGa = selectLatestGaAtOrBelow(releaseTags, target); + + const configurations = [ + { + id: `${compiledVersion}-compiled`, + label: `${compiledVersion} compiled`, + engineVersion: compiledVersion, + surface: 'compiled-simplified', + executionBackend: 'standard', + engineMode: 'legacy', + artifactName: `ppl-lint-observation-${compiledVersion}-compiled`, + exportRuntimeBundle: false, + }, + { + id: 'latest-release-runtime', + label: `Latest release (${latestGa}) runtime`, + engineVersion: latestGa, + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-latest-release-runtime', + exportRuntimeBundle: true, + }, + { + id: 'pr-build-runtime', + label: 'PR runtime', + engineVersion: target.raw, + surface: 'runtime-bundle', + executionBackend: 'standard', + engineMode: 'calcite', + artifactName: 'ppl-lint-observation-pr-build-runtime', + exportRuntimeBundle: true, + }, + ]; + + return { + schemaVersion: 1, + sqlSha, + prTargetVersion: target.raw, + normalizedPrTarget: target.normalized, + latestEligibleGa: latestGa, + osd: { + repository: osdRepository, + ref: osdRef, + }, + configurations, + releasedTargets: { + include: configurations.slice(0, 2).map((configuration) => ({ + version: configuration.engineVersion, + configuration_id: configuration.id, + surface: configuration.surface, + label: configuration.id.endsWith('-compiled') ? 'compiled' : 'runtime', + export_runtime_bundle: configuration.exportRuntimeBundle, + artifact_name: configuration.artifactName, + })), + }, + }; +} + +function main() { + try { + const args = parseArgs(process.argv.slice(2)); + const plan = createPlan({ + buildFile: args['build-file'], + releaseTags: fs.readFileSync(args['release-tags'], 'utf8'), + compiledVersion: args['compiled-version'], + sqlSha: args['sql-sha'], + osdRepository: args['osd-repository'], + osdRef: args['osd-ref'], + }); + fs.mkdirSync(path.dirname(path.resolve(args.out)), { recursive: true }); + fs.writeFileSync(args.out, `${JSON.stringify(plan, null, 2)}\n`); + process.stdout.write(`${JSON.stringify(plan)}\n`); + } catch (error) { + process.stderr.write(`[ppl-lint-plan] ${error.message}\n`); + process.exitCode = 2; + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main(); +} diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 0875f902338..628bd707ff4 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -130,6 +130,31 @@ const SURFACE = (() => { return requested; })(); +const APPLICABLE_ONLY = process.env.PPL_LINT_APPLICABLE_ONLY === '1'; +const ENGINE_MODE = (() => { + const requested = process.env.PPL_LINT_ENGINE_MODE; + if (requested === undefined || requested === '') { + if (APPLICABLE_ONLY) { + // eslint-disable-next-line no-console + console.error( + '[ppl-lint-frontend] FATAL: PPL_LINT_ENGINE_MODE is required when ' + + 'PPL_LINT_APPLICABLE_ONLY=1.' + ); + process.exit(2); + } + return undefined; + } + if (!['calcite', 'legacy'].includes(requested)) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-frontend] FATAL: PPL_LINT_ENGINE_MODE must be "calcite" or "legacy", ` + + `got "${requested}".` + ); + process.exit(2); + } + return requested; +})(); + function log(message) { // eslint-disable-next-line no-console console.log(`[ppl-lint-detector-contract] ${message}`); @@ -533,6 +558,37 @@ function versionMatchesRange(range, version) { return true; } +export function compatibilityExclusion(spec, version, surface, engineMode) { + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== surface) { + return { + reason: 'surface', + detail: `grammarSurface=${contractSurface}, running ${surface}`, + }; + } + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + const have = parseVersion(version); + const min = parseVersion(appliesTo.minVersion); + const max = parseVersion(appliesTo.maxVersion); + if ( + have && + ((min && compareVersion(have, min) < 0) || + (max && compareVersion(have, max) > 0)) + ) { + return { + reason: 'version', + detail: `wiring.appliesTo excludes ${version || 'unknown version'}`, + }; + } + if (appliesTo.engine && appliesTo.engine !== engineMode) { + return { + reason: 'engine', + detail: `wiring.appliesTo.engine=${appliesTo.engine}, running ${engineMode}`, + }; + } + return undefined; +} + /** * Select the single expectation that applies to the candidate version + engine. * Exactly one must match (design §5.3): zero means the rule test does not cover @@ -615,10 +671,10 @@ function checkWiring(spec, catalog, getDetector, failures) { * candidate backend version so version filtering matches the backend, and sets * an enable override for default-off rules that declare `forceEnable`. */ -function buildContext(spec, engineVersion) { +function buildContext(spec, engineVersion, engineMode) { const fc = spec.frontendContext || {}; const context = { - isCalcite: fc.isCalcite !== false, + isCalcite: engineMode ? engineMode === 'calcite' : fc.isCalcite !== false, dataSourceVersion: engineVersion || undefined, // Pin the "latest verified engine" to the candidate version rather than the // hardcoded OSD_KNOWN_VERSION ('3.7.0'), which can mis-filter rules near a @@ -1131,6 +1187,19 @@ function main() { const index = spec.index; const channel = contractChannel(spec); const scoringFailures = reportOnly ? reportOnlyFailures : failures; + if (APPLICABLE_ONLY) { + const exclusion = compatibilityExclusion(spec, engineVersion, surface, ENGINE_MODE); + if (exclusion) { + log(`SKIP ${ruleId} (${exclusion.detail}) — ${path.basename(file)}`); + if (exclusion.reason === 'surface') { + skippedForSurface.push({ + ruleId, + contractSurface: spec.grammarSurface || 'runtime-bundle', + }); + } + continue; + } + } const entry = checkWiring(spec, catalog, getDetector, scoringFailures); if (!entry) { for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { @@ -1187,7 +1256,7 @@ function main() { continue; } - const context = buildContext(spec, engineVersion); + const context = buildContext(spec, engineVersion, ENGINE_MODE); const expectation = selectExpectation(spec, engineVersion, context.isCalcite, scoringFailures, { allowMissing: observeOnly, });