From 3d0b66de5d132664839e9bde1b0899b1202a4c7a Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 15:34:18 -0700 Subject: [PATCH 01/39] test(ppl-lint): add invalid-capture-group-name validation contract Pins the rex capture-group-name rule to live /_plugins/_ppl behavior: an underscore in a capture group name is rejected with IllegalArgumentException (validation introduced in 3.4, #4434), while an alphanumeric name is accepted. Joins the enforced set: the rejection is deterministic and rule-unique, and the control query exercises the same command with a valid name. Signed-off-by: Hanyu Wei --- .../invalid-capture-group-name.spec.json | 62 +++++++++++++++++++ .../ppl-lint/contracts/manifest.json | 2 + 2 files changed, 64 insertions(+) create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json new file mode 100644 index 00000000000..c651af803b6 --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -0,0 +1,62 @@ +{ + "schemaVersion": 3, + "ruleId": "invalid-capture-group-name", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": ["rexCommand", "stringLiteral"], + "wiring": { + "detector": "invalid-capture-group-name", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": false, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { "email": "text" } + }, + "index": "opensearch-sql_test_index_account", + "queries": { + "rex-capture-name-underscore": { + "role": "trigger", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" + }, + "rex-capture-name-alphanumeric-control": { + "role": "control", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email, username, domain | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "engine": "calcite", + "queries": { + "rex-capture-name-underscore": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user_name'." + } + } + } + }, + "rex-capture-name-alphanumeric-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index 805e1be9fd8..bab787584db 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -2,6 +2,7 @@ "schemaVersion": 3, "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus; `enforced` is the phase-one, reviewed, error-severity subset with a stable backend rejection oracle that blocks a PR (design §5.1, §5.2). Everything not in `enforced` runs non-blocking (nightly / advisory) until it has an equally stable oracle and owner review.", "contracts": [ + "invalid-capture-group-name.spec.json", "unsupported-window-function-in-eventstats.spec.json", "division-by-zero.spec.json", "head-without-sort.spec.json", @@ -13,6 +14,7 @@ "replace-wildcard-asymmetry.spec.json" ], "enforced": [ + "invalid-capture-group-name.spec.json", "unsupported-window-function-in-eventstats.spec.json", "multisearch-min-subsearch.spec.json", "union-min-datasets.spec.json", From 479993508f0dabd7057452e94f8b07abfadc3a40 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 15:34:50 -0700 Subject: [PATCH 02/39] feat(ci): validate default-error PPL lint rules across engine versions The existing PPL lint contract validates ONE engine: the build from the pull request. But a lint rule ships to every user, and each user's cluster runs whatever version they run. A rule that is correct on main can be a false positive on 3.6 or a false negative on 3.7, and nothing catches it. This adds a multi-version companion that validates every DEFAULT-ERROR rule (enabled + severity error in OSD's rules_catalog.json -- the diagnostics a user cannot opt out of) against several engine versions at once, and reports what to change in the linter when one disagrees. Caught (a): a per-version matrix. Released legs run the official opensearchproject/opensearch: image, which bundles the matching opensearch-sql plugin, so no old branch is built; the pr-build leg is the same Gradle test cluster the single-version check uses. Every leg runs the SAME contract oracle (PplLintRuleValidationIT) under a new -Dppl.lint.observe.only, which records real behavior instead of asserting -- on an older engine a mismatch IS the signal, not a broken run. Engine floor is 3.6.0, the first release containing GET /_plugins/_ppl/_grammar (#5162); a 3.5 leg could not export a bundle for the detectors to lint. Told (b): scripts/ppl-lint/drift.mjs classifies each disagreement into one of eight drift classes and emits one remediation naming the file to edit -- version-scope-rule (fix appliesTo, or disable the rule), update-detector (the detector regressed, went too broad, or its grammar anchor was renamed), or update-contract (the linter is right; the pinned expectation is stale). A renamed parser rule reports the closest current rule names, once per rule/version rather than once per query. Two guards keep the check from passing vacuously: the detector runner now records the catalog's default-error census, and the aggregate step fails if a rule in it has no contract file -- so a new error rule cannot land unvalidated; and a leg with missing artifacts is a hard failure, never a dropped version. Also closes the last default-error coverage gap: flat-object-subfield had no contract. Adds a flat_object test fixture (the repo had none) and pins all four cases, live-verified on 3.8 -- both a dotted subfield and the bare root are rejected. Note its rejection reason is byte-identical to field-validation's, so attribution comes from the detector's ruleId, not the engine's wording. Non-enforcing for now: the required check stays the single-version validation-result. Promoting this needs a green baseline across the matrix first, so an already-drifted rule does not block unrelated PRs on day one. Verified: 35 classifier/aggregator tests green; observe-only leg run end to end against a live 3.8 cluster (all 7 default-error rules agree); red/green proven by injecting five realistic drifts and confirming five distinct remediations. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 429 ++++++++++++++ .../remote/PplLintRuleValidationIT.java | 85 ++- .../sql/legacy/SQLIntegTestCase.java | 8 + .../org/opensearch/sql/legacy/TestUtils.java | 5 + .../opensearch/sql/legacy/TestsConstants.java | 1 + .../src/test/resources/flat_object.json | 6 + .../flat_object_index_mapping.json | 15 + .../contracts/flat-object-subfield.spec.json | 109 ++++ .../ppl-lint/contracts/manifest.json | 17 +- ...ed-window-function-in-eventstats.spec.json | 1 + scripts/ppl-lint/README.md | 110 +++- .../__tests__/aggregate-versions.test.mjs | 360 ++++++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 399 +++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 554 +++++++++++++++++ scripts/ppl-lint/drift.mjs | 556 ++++++++++++++++++ scripts/ppl-lint/run-frontend-contract.mjs | 10 + 16 files changed, 2654 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/ppl-lint-multiversion-validation.yml create mode 100644 integ-test/src/test/resources/flat_object.json create mode 100644 integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json create mode 100644 integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json create mode 100644 scripts/ppl-lint/__tests__/aggregate-versions.test.mjs create mode 100644 scripts/ppl-lint/__tests__/drift.test.mjs create mode 100644 scripts/ppl-lint/aggregate-versions.mjs create mode 100644 scripts/ppl-lint/drift.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml new file mode 100644 index 00000000000..f4ad09b6a82 --- /dev/null +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -0,0 +1,429 @@ +name: PPL lint multi-version validation + +# Multi-version companion to ppl-lint-rule-validation.yml. +# +# The sibling workflow answers "do the OSD PPL lint detectors and THIS engine +# build agree?". It validates one engine: the one built from the PR. That leaves +# the failure mode that actually reaches users unguarded — a lint rule ships to +# everyone, but each user runs it against whatever engine version their cluster +# happens to be. A rule that is correct on main can be a false positive on 3.6 or +# a false negative on 3.7, and nothing notices. +# +# This workflow validates every DEFAULT-ERROR rule (enabled: true + severity: +# error in OSD's rules_catalog.json) against SEVERAL released engine versions +# plus the PR build, and — when a rule disagrees with any of them — says what to +# change in the linter rather than only that a count was wrong. +# +# Why default-error only: an error-severity rule is one the user cannot opt out +# of and which marks their query as broken. A wrong error is the most expensive +# possible lint defect, so that set gets the multi-version treatment first. +# Warning/info rules stay on the single-version check. The set is not hand-copied: +# the detector run records the catalog's default-error census, and the aggregate +# step fails if a rule in that census has no contract file (see manifest.json's +# `defaultError` note). +# +# Shape — a per-version matrix of observation legs, then one aggregation: +# +# observe (matrix: 3.6.0, 3.7.0, pr-build) ──▶ aggregate ──▶ drift report +# +# Each leg produces the SAME four artifacts the single-version workflow already +# defines (ppl-grammar-bundle.json, target.json, backend-report.json, +# detector-report.json), so this workflow adds no new producer format — only the +# per-version fan-out and the cross-version comparison. +# +# Released legs run the official distribution image, which bundles the matching +# opensearch-sql plugin (verified against opensearch-build's release manifests), +# so no old branch has to be built. The `pr-build` leg is the same Gradle test +# cluster the sibling workflow uses. +# +# Engine floor: 3.6.0. GET /_plugins/_ppl/_grammar landed in #5162 (`fe95703b5`), +# which is an ancestor of the 3.6 release branch but NOT of 3.5 — a 3.5 leg could +# not export a candidate grammar bundle, so the detector half would have nothing +# to lint against. Raise `ENGINE_VERSIONS` as older versions leave support. +# +# Non-enforcing on purpose, for now: it reports and uploads, and the required +# check stays the sibling workflow's `validation-result`. Promoting this to +# required needs a green baseline across the whole matrix first (a rule that has +# quietly drifted on 3.6 would otherwise block every unrelated PR on day one). + +on: + # Nightly is the primary schedule: the matrix pulls three engine images, so it + # is too slow to sit on every push. + schedule: + - cron: '30 10 * * *' + # Run on PRs that touch the contract corpus or this machinery, where the whole + # point is to see the multi-version effect of the change. + pull_request: + paths: + - 'integ-test/src/test/resources/ppl-lint/**' + - 'scripts/ppl-lint/**' + - '.github/workflows/ppl-lint-multiversion-validation.yml' + workflow_dispatch: + inputs: + osd_repo: + description: OSD repository to check out. Defaults to opensearch-project/OpenSearch-Dashboards. + required: false + type: string + osd_ref: + description: OSD commit or branch whose detectors are validated. + required: false + type: string + engine_versions: + description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' + required: false + type: string + +permissions: + contents: read + +env: + # Released engine versions to validate, newest last. Each must be >= 3.6.0 (the + # _grammar endpoint floor) and must have a published distribution image. + ENGINE_VERSIONS: '["3.6.0","3.7.0"]' + +jobs: + # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a + # dependabot bump moves one set of action versions rather than two. + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + # Resolve the matrix and the OSD target once, so every leg and the aggregate + # step agree on exactly what is being validated. + plan: + name: Plan matrix + runs-on: ubuntu-latest + outputs: + released: ${{ steps.plan.outputs.released }} + osd_repo: ${{ steps.plan.outputs.osd_repo }} + osd_ref: ${{ steps.plan.outputs.osd_ref }} + steps: + - name: Resolve engine versions and OSD target + id: plan + env: + REQUESTED_VERSIONS: ${{ inputs.engine_versions }} + DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} + REQUESTED_REPO: ${{ inputs.osd_repo }} + REQUESTED_REF: ${{ inputs.osd_ref }} + VAR_REPO: ${{ vars.OSD_REPO }} + VAR_REF: ${{ vars.OSD_REF }} + run: | + set -euo pipefail + released="${REQUESTED_VERSIONS:-$DEFAULT_VERSIONS}" + # Fail loudly on a malformed override rather than silently validating + # an empty matrix (which would look like a pass). + echo "$released" | python3 -c " + import json,sys + v=json.load(sys.stdin) + assert isinstance(v,list) and v, 'engine_versions must be a non-empty JSON array' + for item in v: + assert isinstance(item,str), 'engine_versions entries must be strings' + " + echo "released=$released" >> "$GITHUB_OUTPUT" + # Same precedence as the sibling workflow: dispatch input, then repo + # variable, then the canonical upstream default. + echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" + echo "osd_ref=${REQUESTED_REF:-${VAR_REF:-main}}" >> "$GITHUB_OUTPUT" + + # One leg per released engine version: run the contract queries against the + # official distribution image (which bundles the matching sql plugin) and + # export that engine's grammar bundle. + observe-released: + name: Observe engine ${{ matrix.version }} + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.released) }} + services: + opensearch: + image: opensearchproject/opensearch:${{ matrix.version }} + env: + discovery.type: single-node + # The lint contract only needs the PPL query and grammar endpoints, so + # run without the security plugin: no TLS or credentials to manage, and + # the observed error bodies are the engine's own rather than a proxy's. + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Wait for the engine and confirm its version + id: engine + run: | + set -euo pipefail + for i in $(seq 1 40); do + if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi + echo "waiting for engine (${i}/40)..." + sleep 5 + done + cat /tmp/root.json + reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") + echo "reported=$reported" >> "$GITHUB_OUTPUT" + # A leg mislabeled as another version would attribute drift to the wrong + # engine, so require the image to be what the matrix asked for. + case "$reported" in + ${{ matrix.version }}*) ;; + *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + esac + # The PPL plugin must actually be present, or every query would "pass" + # by failing identically. + curl -sf http://localhost:9200/_cat/plugins | grep -i sql + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # The same contract oracle the single-version workflow runs, pointed at an + # external cluster instead of a Gradle-managed one. One oracle, many + # engines: a per-version copy would be free to drift from the real check. + - name: Run contract observation against engine ${{ matrix.version }} + run: | + set -euo pipefail + mkdir -p leg + ./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ + -Dppl.lint.grammar.bundle="$(pwd)/leg/ppl-grammar-bundle.json" \ + -Dppl.lint.target="$(pwd)/leg/target.json" + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-${{ matrix.version }} + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-${{ matrix.version }}-logs + path: integ-test/build/reports/** + + # The PR's own engine build, so the newest point in the matrix is the code under + # review rather than the last release. Same oracle as the released legs; the only + # difference is a Gradle-managed cluster instead of a published image, which is + # why it cannot just be another matrix entry. + observe-pr-build: + name: Observe engine pr-build + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # Observe-only here too, so this leg reports what the PR engine does rather + # than duplicating the sibling workflow's assertions. The sibling workflow + # remains the enforcing single-version check. + - name: Run contract observation against the PR build + run: | + set -euo pipefail + mkdir -p leg + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report=$(pwd)/leg/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/leg/target.json" + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-pr-build + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-pr-build-logs + path: | + integ-test/build/reports/** + integ-test/build/testclusters/*/logs/* + + # Lint each engine's exported grammar with the OSD detectors. Separate from the + # observation legs because OSD needs a newer Node/glibc than the engine image + # provides, and because one bootstrap can serve every leg. + # + # `always()` so a single broken leg still yields a report for the others: a + # partial matrix must be visibly partial, not silently absent. The aggregate + # step fails if NO leg produced a report. + detect: + name: Detect on each engine grammar + needs: + - plan + - observe-released + - observe-pr-build + if: ${{ always() && needs.plan.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 40 + outputs: + osd_sha: ${{ steps.osd-rev.outputs.sha }} + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ needs.plan.outputs.osd_repo }} + ref: ${{ needs.plan.outputs.osd_ref }} + path: .ci/OpenSearch-Dashboards + + - name: Record OSD revision + id: osd-rev + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "OSD revision: \`$sha\` (${{ needs.plan.outputs.osd_repo }} @ \`${{ needs.plan.outputs.osd_ref }}\`)" >> "$GITHUB_STEP_SUMMARY" + + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + - name: Download all leg artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: ppl-lint-leg-* + path: legs + + # One detector pass per leg, each against THAT engine's grammar bundle. The + # runner is the same SQL-owned script the single-version workflow uses, so + # the detector half cannot drift between the two checks. + - name: Run detectors against every engine grammar + working-directory: .ci/OpenSearch-Dashboards + run: | + set -euo pipefail + shopt -s nullglob + legs=("$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*) + if [ ${#legs[@]} -eq 0 ]; then + echo "::error::no leg artifacts were downloaded; nothing to validate." + exit 1 + fi + for leg in "${legs[@]}"; do + # Skip the log-only artifacts an observation failure may have uploaded. + [ -f "$leg/ppl-grammar-bundle.json" ] || { echo "skipping $leg (no grammar bundle)"; continue; } + version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') + echo "=== detectors vs engine $version ===" + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json" \ + PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ + PPL_LINT_REPORT="$leg/detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + > "$leg/detector.log" 2>&1 || true + # A per-leg non-zero exit is EXPECTED when that engine disagrees with + # the pinned expectation — that is the drift this workflow exists to + # report, and the aggregate step below is what classifies it. Only a + # missing report means the runner itself broke. + if [ ! -f "$leg/detector-report.json" ]; then + echo "::error::detector runner produced no report for engine $version" + tail -50 "$leg/detector.log" || true + exit 1 + fi + tail -5 "$leg/detector.log" || true + done + + # Compare every engine version against every other and against the pinned + # contracts, then print the remediation report. + - name: Aggregate drift across engine versions + id: aggregate + run: | + set -euo pipefail + shopt -s nullglob + args=() + for leg in "$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*; do + [ -f "$leg/detector-report.json" ] || continue + version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') + args+=(--leg "$version=$leg") + done + if [ ${#args[@]} -eq 0 ]; then + echo "::error::no complete legs to aggregate." + exit 1 + fi + node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ + --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + --out "$GITHUB_WORKSPACE/drift-report.json" \ + --summary "$GITHUB_STEP_SUMMARY" \ + "${args[@]}" + + - name: Upload drift report + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-multiversion-drift + path: | + drift-report.json + legs/**/detector-report.json + legs/**/detector.log + legs/**/target.json diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 4185511ab04..ae74fdd2515 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -73,6 +73,24 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { /** Which contracts to run this session; PR is the fast blocking subset. */ private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); + /** + * Observe-only mode, used by the multi-version workflow ({@code + * .github/workflows/ppl-lint-multiversion-validation.yml}). + * + *

The default mode asserts each case against the expectation pinned for the cluster's version, + * which is right when the cluster IS the build under test. The multi-version matrix instead + * points this suite at OLDER released engines, where a disagreement is the very signal being + * collected — a 3.6 engine that accepts what the contract pins as rejected is a finding for the + * drift classifier, not a broken test run. + * + *

So in observe-only mode the suite still runs every query and records the true observed + * behavior in the report, but does not fail on an expectation mismatch, and does not require an + * expectation to exist for this version at all. Failures that mean the RUN itself is broken (no + * grammar bundle, an unreachable cluster, a malformed contract) still fail, because those would + * otherwise produce an empty report that reads as agreement. + */ + private final boolean observeOnly = Boolean.getBoolean("ppl.lint.observe.only"); + private int[] clusterVersion; private String engineVersionRaw; @@ -82,7 +100,20 @@ public void init() throws Exception { enableCalcite(); // Seed the union of every index every scheduled contract needs, once. for (String indexEnum : requiredIndexEnums()) { - loadIndex(Index.valueOf(indexEnum)); + try { + loadIndex(Index.valueOf(indexEnum)); + } catch (Exception e) { + if (!observeOnly) { + throw e; + } + // In the multi-version matrix an older engine may not support a field type + // a fixture uses (a mapping that only exists in a later release). Losing + // that one index must not abort the whole leg — the contracts that need it + // will surface as their own observations, while every other rule is still + // validated against this engine. + System.err.println( + "[ppl-lint] could not seed index " + indexEnum + " on this engine: " + e.getMessage()); + } } clusterVersion = fetchClusterVersion(); } @@ -127,7 +158,14 @@ private void runContract( try { JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); if (selected == null) { - return; // no/ambiguous version expectation — failure already recorded. + if (!observeOnly) { + return; // no/ambiguous version expectation — failure already recorded. + } + // Observe-only: an engine the corpus does not pin is exactly what the + // multi-version matrix wants to learn about, so record the raw behavior of + // every query and let the aggregator decide whether the gap matters. + observeAllQueries(ruleId, index, queries, report); + return; } JSONObject expectedQueries = selected.getJSONObject("queries"); for (String queryName : expectedQueries.keySet()) { @@ -153,9 +191,16 @@ private void runContract( entry.put("outcome", "pass"); log(ruleId, queryName, "PASS (" + kind + ", " + role + ")"); } catch (AssertionError | RuntimeException e) { - entry.put("outcome", "fail").put("error", String.valueOf(e.getMessage())); - failures.add("[" + ruleId + "/" + queryName + "] " + e.getMessage()); - log(ruleId, queryName, "FAIL (" + kind + "): " + e.getMessage()); + entry.put("outcome", observeOnly ? "observed-mismatch" : "fail"); + entry.put("error", String.valueOf(e.getMessage())); + if (observeOnly) { + // Not a failure here: the observation is the deliverable, and the + // drift classifier turns it into a remediation. + log(ruleId, queryName, "OBSERVED MISMATCH (" + kind + "): " + e.getMessage()); + } else { + failures.add("[" + ruleId + "/" + queryName + "] " + e.getMessage()); + log(ruleId, queryName, "FAIL (" + kind + "): " + e.getMessage()); + } } report.put(entry); } @@ -164,6 +209,36 @@ private void runContract( } } + /** + * Observe-only helper: run every query a contract declares and record what the engine actually + * did, without comparing against any expectation. Used when this engine version has no matching + * {@code expectations[]} entry, so the multi-version report still shows real behavior instead of + * a blank row that would read as agreement. + */ + private void observeAllQueries( + String ruleId, String index, JSONObject queries, JSONArray report) { + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject entry = reportEntry(ruleId, queryName, role, query, "observe-only"); + try { + BackendObservation obs = observeBackend(query); + entry + .put("rejected", obs.rejected) + .put("observed", obs.toJson()) + .put("outcome", "observed"); + log(ruleId, queryName, "OBSERVED (" + (obs.rejected ? "rejected" : "accepted") + ")"); + } catch (IOException | RuntimeException e) { + // A transport-level problem is a broken run, not an engine verdict; mark it + // so the aggregator does not read the absence of a rejection as acceptance. + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR: " + e.getMessage()); + } + report.put(entry); + } + } + /** * Select the single {@code expectations[]} entry that applies to the candidate backend version * and engine. Exactly one must match: zero means the rule test does not cover this version diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index fc15c908c63..38e37c41d31 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -869,6 +869,14 @@ public enum Index { "flattened_value", null, "src/test/resources/flattened_value.json"), + // An index with a `flat_object` field, which PPL cannot reference at all — + // neither the root nor a dotted subfield. Backs the flat-object-subfield lint + // contract; see ppl-lint/contracts/flat-object-subfield.spec.json. + FLAT_OBJECT( + TestsConstants.TEST_INDEX_FLAT_OBJECT, + "flat_object", + getFlatObjectIndexMapping(), + "src/test/resources/flat_object.json"), DUPLICATION_NULLABLE( TestsConstants.TEST_INDEX_DUPLICATION_NULLABLE, "duplication_nullable", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index c478165bf07..198527d1efc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -523,6 +523,11 @@ public static String getAccountExtendedIndexMapping() { return getMappingFile(mappingFile); } + public static String getFlatObjectIndexMapping() { + String mappingFile = "flat_object_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getPhraseIndexMapping() { String mappingFile = "phrase_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index 5d7eeb328af..957ff0108d6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -76,6 +76,7 @@ public class TestsConstants { public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test"; public static final String TEST_INDEX_ALIAS = TEST_INDEX + "_alias"; public static final String TEST_INDEX_FLATTENED_VALUE = TEST_INDEX + "_flattened_value"; + public static final String TEST_INDEX_FLAT_OBJECT = TEST_INDEX + "_flat_object"; public static final String TEST_INDEX_GEOIP = TEST_INDEX + "_geoip"; public static final String DATASOURCES = ".ql-datasources"; public static final String TEST_INDEX_STATE_COUNTRY = TEST_INDEX + "_state_country"; diff --git a/integ-test/src/test/resources/flat_object.json b/integ-test/src/test/resources/flat_object.json new file mode 100644 index 00000000000..03ed0d15d67 --- /dev/null +++ b/integ-test/src/test/resources/flat_object.json @@ -0,0 +1,6 @@ +{"index":{"_id":"1"}} +{"name":"alpha","status":200,"attributes":{"service":"checkout","region":"us-east-1"}} +{"index":{"_id":"2"}} +{"name":"beta","status":500,"attributes":{"service":"search","region":"us-west-2"}} +{"index":{"_id":"3"}} +{"name":"gamma","status":200,"attributes":{"service":"checkout","region":"eu-west-1"}} diff --git a/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json new file mode 100644 index 00000000000..5721d2c3773 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/flat_object_index_mapping.json @@ -0,0 +1,15 @@ +{ + "mappings": { + "properties": { + "name": { + "type": "keyword" + }, + "status": { + "type": "integer" + }, + "attributes": { + "type": "flat_object" + } + } + } +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json new file mode 100644 index 00000000000..25ac4cf66cf --- /dev/null +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -0,0 +1,109 @@ +{ + "schemaVersion": 3, + "ruleId": "flat-object-subfield", + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "requiredParserRules": ["qualifiedName", "wcQualifiedName"], + "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true).", + "wiring": { + "detector": "flat-object-subfield", + "enabled": true, + "severity": "error", + "runtimeOnly": false, + "needsContext": true, + "needsExplain": false, + "appliesTo": {} + }, + "backendFixture": { + "indices": ["FLAT_OBJECT"], + "clusterSettings": { "calcite": true, "calciteFallback": false } + }, + "frontendContext": { + "isCalcite": true, + "deriveFromMapping": { + "name": "keyword", + "status": "integer", + "attributes": "flat_object" + } + }, + "index": "opensearch-sql_test_index_flat_object", + "queries": { + "flat-object-dotted-subfield": { + "role": "trigger", + "query": "source={{index}} | fields attributes.service" + }, + "flat-object-bare-root": { + "role": "trigger", + "query": "source={{index}} | fields attributes" + }, + "flat-object-in-where": { + "role": "trigger", + "query": "source={{index}} | where attributes.service = 'checkout'" + }, + "non-flat-field-control": { + "role": "control", + "query": "source={{index}} | fields name, status | head 1" + } + }, + "expectations": [ + { + "version": ">=3.4.0", + "engine": "calcite", + "queries": { + "flat-object-dotted-subfield": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } + } + } + }, + "flat-object-bare-root": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes] not found." + } + } + } + }, + "flat-object-in-where": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } + } + } + }, + "non-flat-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { "datarowsNonEmpty": true } + } + } + } + } + ] +} diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index bab787584db..a8b031315d6 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -8,6 +8,7 @@ "head-without-sort.spec.json", "disabled-join-type.spec.json", "field-validation.spec.json", + "flat-object-subfield.spec.json", "dedup-consecutive-unsupported.spec.json", "multisearch-min-subsearch.spec.json", "union-min-datasets.spec.json", @@ -20,9 +21,16 @@ "union-min-datasets.spec.json", "replace-wildcard-asymmetry.spec.json" ], - "pendingReview": [ - "field-validation.spec.json" + "defaultError": [ + "invalid-capture-group-name.spec.json", + "unsupported-window-function-in-eventstats.spec.json", + "multisearch-min-subsearch.spec.json", + "union-min-datasets.spec.json", + "replace-wildcard-asymmetry.spec.json", + "field-validation.spec.json", + "flat-object-subfield.spec.json" ], + "pendingReview": [], "nonEnforcing": [ "division-by-zero.spec.json", "head-without-sort.spec.json", @@ -30,8 +38,9 @@ "dedup-consecutive-unsupported.spec.json" ], "notes": { - "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required validation-result check.", - "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. field-validation self-suppresses without field context and is a semantic rule rather than a clean HTTP-400 grammar rejection.", + "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required single-version validation-result check.", + "defaultError": "Every rule that ships enabled at ERROR severity in the OSD catalog — the set the MULTI-VERSION check enforces (scripts/ppl-lint/aggregate-versions.mjs). A default-error rule is what users cannot opt out of and what blocks a query in the editor, so it is exactly the set that must agree with every supported engine version. Kept in sync with the catalog by the coverage assertion in the aggregate step: a rules_catalog.json entry with enabled:true + severity:error and no contract file here fails the check.", + "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. Empty now that field-validation and flat-object-subfield are pinned across versions by the multi-version check; they remain outside single-version `enforced` because their backend oracle is a semantic 'Field [...] not found.' rejection shared with each other rather than a rule-unique grammar rejection.", "nonEnforcing": "Warning / info / advisory / result-shape rules. They lack a stable backend rejection oracle and never block a PR; they run for coverage on the nightly schedule." } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index e1254b14583..6f6fbd313b0 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -1,6 +1,7 @@ { "schemaVersion": 3, "ruleId": "unsupported-window-function-in-eventstats", + "detectorPath": "packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts", "grammarSurface": "compiled-simplified", "schedule": "pr", "wiring": { diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index e9578517608..a78b1f5cdcc 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -170,14 +170,120 @@ rule cannot be validated end to end. `manifest.json` partitions the corpus: - `enforced` — reviewed error rules with a deterministic backend rejection and a - valid negative control. These block `validation-result`. Phase one: + valid negative control. These block `validation-result` on the single-version + check: `invalid-capture-group-name`, `unsupported-window-function-in-eventstats`, `multisearch-min-subsearch`, `union-min-datasets`, `replace-wildcard-asymmetry`. +- `defaultError` — every rule that ships **enabled at error severity** in OSD's + `rules_catalog.json`. This is the set the **multi-version** check enforces (see + below). It is a superset of `enforced`, adding `field-validation` and + `flat-object-subfield`. - `pendingReview` — error rules awaiting Peng/Chen usefulness review before - joining `enforced` (currently `field-validation`). + joining `enforced`. Empty: `field-validation` and `flat-object-subfield` are now + pinned across versions by the multi-version check, but stay out of the + single-version `enforced` set because their backend oracle is a semantic + `Field [...] not found.` rejection they share with each other rather than a + rule-unique grammar rejection. - `nonEnforcing` — warning/info/advisory/result-shape rules. They run on the nightly schedule for coverage and never block a PR. +## Multi-version validation + +The check above validates **one** engine: the build from the PR. But a lint rule +ships to every user, and each user's cluster is on whatever version they run. A +rule that is correct on `main` can be a false positive on 3.6 or a false negative +on 3.7, and the single-version check cannot see it. + +[`ppl-lint-multiversion-validation.yml`](../../.github/workflows/ppl-lint-multiversion-validation.yml) +validates every `defaultError` rule against several engine versions at once, and +reports **what to change in the linter** when one disagrees. + +``` +observe (matrix: 3.6.0, 3.7.0 released images + pr-build) + └── each leg exports the same 4 artifacts as the single-version check +detect (one OSD bootstrap, one detector pass per leg's grammar) + └── aggregate-versions.mjs → drift-report.json + remediation report +``` + +Released legs run the official `opensearchproject/opensearch:` image, +which bundles the matching `opensearch-sql` plugin, so no old branch is built. The +`pr-build` leg is the same Gradle test cluster the single-version check uses. Both +run the **same** contract oracle (`PplLintRuleValidationIT`) with +`-Dppl.lint.observe.only=true`, which records real behavior instead of asserting +against expectations — on an older engine a mismatch is the signal being +collected, not a broken run. + +**Engine floor: 3.6.0.** `GET /_plugins/_ppl/_grammar` landed in #5162, which is +an ancestor of 3.6 but not 3.5, so a 3.5 leg could not export a grammar bundle for +the detectors to lint against. + +This workflow is **non-enforcing for now**: it reports and uploads, while the +required check stays the single-version `validation-result`. Promoting it needs a +green baseline across the whole matrix first, so a rule that has already drifted +on 3.6 does not block every unrelated PR on day one. + +### What a drift report tells you + +Every finding names a drift class, the evidence, and one remediation action: + +| Action | When | What you change | +| --- | --- | --- | +| `version-scope-rule` | the engine relaxed (or never had) the behavior on some versions | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` — or `enabled: false` if no supported engine rejects it any more | +| `update-detector` | the detector regressed, went too broad, or its grammar anchor was renamed | the rule's detector `.ts` (named in the finding) | +| `update-contract` | the linter is right and only the pinned expectation is stale | the `expectations[]` entry for that version | + +Drift classes: `grammar-rule-missing` (a parser rule the detector walks was +renamed or removed — the finding names the closest current rule names), +`engine-relaxed` / `engine-tightened` (the engine's verdict flipped), +`engine-message-changed` (same verdict, reworded error), `detector-silent` / +`detector-noisy` (false negative / false positive), `version-scope-too-narrow` +(the engine rejects but the rule is scoped away from that version, so users see no +diagnostic), and `severity-mismatch`. + +Two guards keep the check from passing vacuously: + +- A rule that is default-error in OSD's catalog but has no contract file fails the + run. The detector runner records the catalog's default-error census in + `detector-report.json`, and the aggregate step compares it against + `manifest.defaultError` — so a new error rule cannot land unvalidated. +- A leg whose artifacts are missing is a hard failure, never a silently dropped + version. + +A rule that is out of scope on an engine (`appliesTo` excludes it) and that the +engine also accepts is reported as `n/a (out of scope)`, not as drift — that is +the version window working. But if the engine *rejects* the trigger there, it is +`version-scope-too-narrow`. + +### Running the multi-version check locally + +Each leg needs a reachable cluster. Point the observe step at any running engine: + +```bash +# Observe one engine (repeat per version into its own leg dir). +mkdir -p legs/3.7.0 +./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true \ + -Dppl.lint.report=$PWD/legs/3.7.0/backend-report.json \ + -Dppl.lint.grammar.bundle=$PWD/legs/3.7.0/ppl-grammar-bundle.json \ + -Dppl.lint.target=$PWD/legs/3.7.0/target.json + +# Lint each leg's grammar from an OSD checkout (writes detector-report.json), +# then compare every version at once: +node scripts/ppl-lint/aggregate-versions.mjs \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 \ + --out drift-report.json +``` + +The classifier is pure and has no cluster or OSD dependency, so its tests run +anywhere: + +```bash +node --test "scripts/ppl-lint/__tests__/*.test.mjs" +``` + ## Interpreting a failure | Failure | Meaning | diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs new file mode 100644 index 00000000000..8f1bd7493a5 --- /dev/null +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -0,0 +1,360 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Integration tests for the multi-version aggregator. + * + * node --test scripts/ppl-lint/__tests__/aggregate-versions.test.mjs + * + * These drive the real script as a child process over synthetic leg directories + * (the four artifact files each engine leg produces), so they cover the parts the + * pure classifier tests cannot: argument handling, artifact loading, the + * in-scope/out-of-scope split, coverage holes, once-per-rule grammar drift, and + * the process exit code that makes the CI check red or green. + */ + +import assert from 'node:assert/strict'; +import { after, test } from 'node:test'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'aggregate-versions.mjs'); + +/** Contract used by every case: a >=3.7 calcite-only rule with one trigger + one control. */ +const SPEC = { + schemaVersion: 3, + ruleId: 'union-min-datasets', + grammarSurface: 'runtime-bundle', + schedule: 'pr', + requiredParserRules: ['unionCommand'], + wiring: { + detector: 'union-min-datasets', + enabled: true, + severity: 'error', + runtimeOnly: true, + appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, + }, + index: 'test-index', + queries: { + trigger: { role: 'trigger', query: 'union [ source={{index}} ]' }, + control: { role: 'control', query: 'union [ source={{index}} ] [ source={{index}} ]' }, + }, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + trigger: { + detectorCount: 1, + severity: 'error', + backend: { + kind: 'rejection', + httpStatus: 400, + body: { + status: 400, + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }, + control: { detectorCount: 0, backend: { kind: 'result-shape', httpStatus: 200 } }, + }, + }, + ], +}; + +const REJECTION = { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', +}; + +const tmpDirs = []; + +function makeTmp(prefix) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tmpDirs.push(dir); + return dir; +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +/** Write a contract dir holding SPEC (optionally patched) and a manifest. */ +function writeContracts(patch = {}) { + const dir = makeTmp('ppl-lint-contracts-'); + const spec = { ...SPEC, ...patch }; + fs.writeFileSync(path.join(dir, 'union.spec.json'), JSON.stringify(spec)); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 3, + contracts: ['union.spec.json'], + defaultError: ['union.spec.json'], + }) + ); + return dir; +} + +/** + * Write one engine leg. `cases` maps query name to + * { detector: , severities, rejected, type, reason }. + */ +function writeLeg({ + version, + cases, + parserRuleNames = ['unionCommand', 'unionDataset'], + defaultErrorRules, +}) { + const dir = makeTmp(`ppl-lint-leg-${version}-`); + fs.writeFileSync( + path.join(dir, 'target.json'), + JSON.stringify({ engineVersion: version, grammarHash: `sha256:${version}` }) + ); + fs.writeFileSync( + path.join(dir, 'ppl-grammar-bundle.json'), + JSON.stringify({ parserRuleNames }) + ); + + const results = []; + const backend = []; + for (const [queryName, c] of Object.entries(cases)) { + const role = queryName === 'control' ? 'control' : 'trigger'; + results.push({ + ruleId: SPEC.ruleId, + queryName, + role, + expected: role === 'trigger' ? 1 : 0, + actual: c.detector, + severities: c.severities || (c.detector > 0 ? ['error'] : []), + }); + backend.push({ + ruleId: SPEC.ruleId, + queryName, + role, + rejected: !!c.rejected, + observed: { + httpStatus: c.rejected ? 400 : 200, + rejected: !!c.rejected, + ...(c.rejected ? { type: c.type || REJECTION.type, reason: c.reason || REJECTION.reason } : {}), + }, + }); + } + fs.writeFileSync( + path.join(dir, 'detector-report.json'), + JSON.stringify({ results, ...(defaultErrorRules ? { defaultErrorRules } : {}) }) + ); + fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); + return dir; +} + +/** Run the aggregator; returns { status, stdout, report }. */ +function run({ contracts, legs, extraArgs = [] }) { + const outDir = makeTmp('ppl-lint-out-'); + const out = path.join(outDir, 'drift-report.json'); + const args = [SCRIPT, '--contracts', contracts, '--out', out]; + for (const [version, dir] of Object.entries(legs)) { + args.push('--leg', `${version}=${dir}`); + } + args.push(...extraArgs); + const result = spawnSync(process.execPath, args, { encoding: 'utf8' }); + const report = fs.existsSync(out) ? JSON.parse(fs.readFileSync(out, 'utf8')) : undefined; + return { status: result.status, stdout: result.stdout || '', stderr: result.stderr || '', report }; +} + +/** The all-agree case, reused as the base for each drift scenario. */ +function healthyLegs() { + return { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + '3.8.0': writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + }; +} + +test('all versions agreeing exits 0 and reports no drift', () => { + const { status, report, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); + assert.equal(status, 0); + assert.equal(report.result.passed, true); + assert.equal(report.drifts.length, 0); + assert.match(stdout, /agrees with all 2 engine version\(s\)/); + // Every rule/version pair is accounted for in the matrix. + assert.equal(report.matrix.length, 2); + assert.ok(report.matrix.every((m) => m.status === 'agree')); +}); + +test('a version where only one engine relaxed is red, and names just that version', () => { + const legs = healthyLegs(); + // 3.8 now accepts what 3.7 still rejects, while the detector keeps flagging. + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.enforcedDriftCount, 1); + const drift = report.drifts[0]; + assert.equal(drift.driftClass, 'engine-relaxed'); + assert.equal(drift.version, '3.8.0'); + assert.equal(drift.remediation.action, 'version-scope-rule'); + // The healthy version is still reported as agreeing. + assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); +}); + +test('a rule out of scope on an older engine that accepts is not drift', () => { + const legs = healthyLegs(); + // 3.6 predates the rule's minVersion and accepts the query: intended silence. + legs['3.6.0'] = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.matrix.find((m) => m.version === '3.6.0').status, 'out-of-scope'); + assert.equal(report.coverageHoles.length, 0); +}); + +test('an out-of-scope engine that rejects is flagged as scoped too narrowly', () => { + const legs = healthyLegs(); + legs['3.6.0'] = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const drift = report.drifts.find((d) => d.version === '3.6.0'); + assert.equal(drift.driftClass, 'version-scope-too-narrow'); + assert.equal(drift.remediation.action, 'version-scope-rule'); +}); + +test('an in-scope version with no expectation is a coverage hole, not silent success', () => { + // The rule applies from 3.7 up, but the contract only pins <3.8 — so a 3.8 + // engine runs a shipped default-error rule with nothing pinning it. + const contracts = writeContracts({ + expectations: [{ ...SPEC.expectations[0], version: '>=3.7.0 <3.8.0' }], + }); + const { status, report } = run({ contracts, legs: healthyLegs() }); + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 1); + const hole = report.coverageHoles[0]; + assert.equal(hole.version, '3.8.0'); + assert.equal(hole.enforced, true); + assert.match(report.matrix.find((m) => m.version === '3.8.0').status, /uncovered/); +}); + +test('a renamed parser rule is reported once per version, not once per query', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + parserRuleNames: ['unionStatement', 'unionDataset'], + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const grammarDrifts = report.drifts.filter((d) => d.driftClass === 'grammar-rule-missing'); + assert.equal(grammarDrifts.length, 1, 'one grammar finding per rule/version'); + assert.equal(grammarDrifts[0].remediation.action, 'update-detector'); + assert.match(grammarDrifts[0].remediation.detail, /unionStatement/); +}); + +test('a silent detector on an unchanged engine is update-detector, never a re-pin', () => { + const legs = healthyLegs(); + legs['3.8.0'] = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + const drift = report.drifts.find((d) => d.version === '3.8.0'); + assert.equal(drift.driftClass, 'detector-silent'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('the leg label is corrected to the engine self-reported version', () => { + // Ask for 3.7.0 but hand over an engine that says 3.8.0: results must be + // attributed to what actually ran. + const legs = { '3.7.0': writeLeg({ version: '3.8.0', cases: { trigger: { detector: 1, rejected: true } } }) }; + const { report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(report.legs[0].label, '3.7.0'); + assert.equal(report.legs[0].engineVersion, '3.8.0'); + assert.match(stdout, /reported engineVersion "3\.8\.0"/); +}); + +test('a missing leg artifact fails loudly instead of dropping the version', () => { + const emptyLeg = makeTmp('ppl-lint-empty-leg-'); + const { status, stderr } = run({ contracts: writeContracts(), legs: { '3.8.0': emptyLeg } }); + assert.equal(status, 2, 'a broken matrix must not be able to pass'); + assert.match(stderr, /expected file not found/); +}); + +test('a default-error rule with no contract file fails the check', () => { + // OSD started shipping `brand-new-error-rule` enabled at error severity, but no + // contract pins it — so no engine version validates it. That must be red, not + // silently absent from the matrix. + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + defaultErrorRules: ['union-min-datasets', 'brand-new-error-rule'], + }), + }; + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 1); + assert.equal(report.result.missingContractCount, 1); + assert.equal(report.missingContracts[0].ruleId, 'brand-new-error-rule'); + assert.match(stdout, /Unvalidated default-error rules/); + assert.match(stdout, /brand-new-error-rule.*no contract file/s); +}); + +test('a census matching the manifest keeps the check green', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + defaultErrorRules: ['union-min-datasets'], + }), + }; + const { status, report } = run({ contracts: writeContracts(), legs }); + assert.equal(status, 0); + assert.equal(report.result.missingContractCount, 0); +}); + +test('a legacy detector report without a census warns instead of failing', () => { + // Older detector builds do not emit defaultErrorRules; the aggregator must say + // so out loud rather than quietly reporting full coverage. + const { status, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); + assert.equal(status, 0); + assert.match(stdout, /no detector leg reported a defaultErrorRules census/); +}); + +test('a bad --leg argument is rejected', () => { + const result = spawnSync( + process.execPath, + [SCRIPT, '--contracts', writeContracts(), '--leg', 'no-equals-sign'], + { encoding: 'utf8' } + ); + assert.equal(result.status, 2); + assert.match(result.stderr, /--leg expects =

/); +}); + +test('at least one leg is required', () => { + const result = spawnSync(process.execPath, [SCRIPT, '--contracts', writeContracts()], { + encoding: 'utf8', + }); + assert.equal(result.status, 2); + assert.match(result.stderr, /at least one --leg/); +}); diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs new file mode 100644 index 00000000000..63640958473 --- /dev/null +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -0,0 +1,399 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the multi-version drift classifier. + * + * Run with plain Node (no Gradle, no cluster, no OSD checkout): + * + * node --test scripts/ppl-lint/__tests__/drift.test.mjs + * + * The classifier is the part of the multi-version contract that decides what an + * engineer is told to do, so every drift class and every remediation branch is + * pinned here. Observations are hand-written rather than gathered from a + * cluster; the live-engine plumbing is exercised by the CI workflow itself. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + DRIFT_CLASSES, + REMEDIATIONS, + classifyDrift, + formatDriftReport, + parseVersion, + suggestParserRules, + versionInAppliesTo, +} from '../drift.mjs'; + +/** A trigger case that agrees on all three sides, used as the mutation base. */ +function agreeingTrigger(overrides = {}) { + return { + ruleId: 'union-min-datasets', + version: '3.7.0', + queryName: 'union-single-dataset', + role: 'trigger', + query: 'union [ source=t ]', + expected: { detectorCount: 1, severity: 'error', backendKind: 'rejection' }, + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + wiring: { appliesTo: { minVersion: '3.7.0', engine: 'calcite' }, runtimeOnly: true }, + ...overrides, + }; +} + +/** A control case that agrees on all three sides. */ +function agreeingControl(overrides = {}) { + return { + ruleId: 'union-min-datasets', + version: '3.7.0', + queryName: 'union-two-datasets-control', + role: 'control', + query: 'union [ source=t ] [ source=t ]', + expected: { detectorCount: 0, backendKind: 'result-shape' }, + observed: { detectorCount: 0, severities: [], backendRejected: false }, + wiring: { appliesTo: { minVersion: '3.7.0', engine: 'calcite' } }, + ...overrides, + }; +} + +// --- the quiet path ----------------------------------------------------------- + +test('agreement produces no drift', () => { + assert.equal(classifyDrift(agreeingTrigger()), null); + assert.equal(classifyDrift(agreeingControl()), null); +}); + +test('a rule out of version scope on an engine that also accepts is silent', () => { + // union-min-datasets does not apply below 3.7, and a 3.6 engine that accepts + // the query is not drift — it is the reason the version window exists. + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', + observed: { detectorCount: 0, severities: [], backendRejected: false }, + }) + ); + assert.equal(drift, null); +}); + +// --- grammar moved ------------------------------------------------------------ + +test('a missing parser rule is reported as update-detector, not as a silent detector', () => { + const drift = classifyDrift( + agreeingTrigger({ + requiredParserRules: ['unionCommand', 'unionDataset'], + parserRuleNames: ['unionStatement', 'unionDataset', 'pplCommands'], + observed: { detectorCount: 0, severities: [], backendRejected: true }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.GRAMMAR_RULE_MISSING); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.target, /union_min_datasets\.ts$/); + // The likely rename is named so the engineer does not diff 259 rule names. + assert.match(drift.remediation.detail, /unionStatement/); + assert.match(drift.evidence, /no parser rule/); +}); + +test('a contract can pin a detector path that breaks the naming convention', () => { + // unsupported-window-function-in-eventstats lives in + // unsupported_window_function.ts, so the derived name would not exist. + const drift = classifyDrift( + agreeingTrigger({ + ruleId: 'unsupported-window-function-in-eventstats', + detectorPath: 'packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts', + wiring: { appliesTo: {} }, + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'CalciteUnsupportedException', + backendReason: 'Unexpected window function: rank', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal( + drift.remediation.target, + 'packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts' + ); +}); + +test('grammar-rule check is skipped when the contract declares no required rules', () => { + const drift = classifyDrift( + agreeingTrigger({ parserRuleNames: ['somethingElse'], requiredParserRules: undefined }) + ); + assert.equal(drift, null); +}); + +// --- engine behavior flips ---------------------------------------------------- + +test('engine relaxation with a still-firing detector demands version scoping', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: false, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + // Both escape hatches are spelled out: bound the version, or disable outright. + assert.match(drift.remediation.detail, /maxVersion/); + assert.match(drift.remediation.detail, /"enabled": false/); + assert.match(drift.evidence, /now ACCEPTS/); +}); + +test('engine relaxation with an already-silent detector only needs a re-pin', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { detectorCount: 0, severities: [], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); + assert.match(drift.remediation.detail, /no linter change/); +}); + +test('engine tightening on a control with a silent detector is a false negative', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command now requires matching schemas.', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_TIGHTENED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /false NEGATIVE/); + assert.match(drift.evidence, /matching schemas/); +}); + +test('engine tightening the detector already catches only needs a re-pin', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'nope', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_TIGHTENED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +// --- wording drift ------------------------------------------------------------ + +test('a reworded rejection is update-contract and points at quoted copy', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'union requires >= 2 datasets, got 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); + assert.match(drift.evidence, /error\.reason/); + assert.match(drift.remediation.detail, /quotes the old engine wording/); +}); + +test('a changed exception type is reported even when the reason is unchanged', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED); + assert.match(drift.evidence, /error\.type/); +}); + +// --- detector-only disagreement ---------------------------------------------- + +test('a silent detector on an unchanged engine names the three silent-failure causes', () => { + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /runtimeOnly/); + assert.match(drift.remediation.detail, /typeMap/); + // Guard the anti-vacuous instruction: never silence the contract instead. + assert.match(drift.remediation.detail, /Do NOT re-pin/); +}); + +test('a noisy detector the engine disagrees with is a false positive', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { detectorCount: 2, severities: ['error', 'error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.remediation.detail, /false positive/); +}); + +test('a noisy detector the engine agrees with points at the expectation', () => { + const drift = classifyDrift( + agreeingControl({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'bad query', + }, + // Engine tightening is the more specific story when the pinned kind is not + // a rejection, so pin the kind as rejection to isolate the noisy branch. + expected: { detectorCount: 0, backendKind: 'rejection' }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +// --- severity ---------------------------------------------------------------- + +test('a downgraded severity is caught even when the count is right', () => { + const drift = classifyDrift( + agreeingTrigger({ observed: { ...agreeingTrigger().observed, severities: ['warning'] } }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.SEVERITY_MISMATCH); + assert.match(drift.remediation.detail, /Restore "union-min-datasets"\.severity/); +}); + +// --- version scoping -------------------------------------------------------- + +test('an out-of-scope rule on an engine that rejects is scoped too narrowly', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', // below the rule's 3.7 minVersion + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'Union command requires at least two datasets. Provided: 1', + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + assert.match(drift.remediation.detail, /minVersion/); +}); + +// --- helpers ----------------------------------------------------------------- + +test('version parsing tolerates snapshot and short forms', () => { + assert.deepEqual(parseVersion('3.8.0-SNAPSHOT'), [3, 8, 0]); + assert.deepEqual(parseVersion('3.7'), [3, 7, 0]); + assert.equal(parseVersion(''), undefined); + assert.equal(parseVersion(undefined), undefined); +}); + +test('appliesTo bounds are inclusive and open-ended when absent', () => { + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, '3.7.0'), true); + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, '3.6.9'), false); + assert.equal(versionInAppliesTo({ maxVersion: '3.8.0' }, '3.8.0'), true); + assert.equal(versionInAppliesTo({ maxVersion: '3.8.0' }, '3.9.0'), false); + assert.equal(versionInAppliesTo({}, '3.9.0'), true); + // An unparseable engine version must never silently drop coverage. + assert.equal(versionInAppliesTo({ minVersion: '3.7.0' }, 'weird-build'), true); +}); + +test('rename suggestions prefer containment then near spellings', () => { + assert.deepEqual(suggestParserRules('unionCommand', ['unionCommandNew', 'zzz'], 3), [ + 'unionCommandNew', + ]); + assert.deepEqual(suggestParserRules('rexCommand', ['regexCommand'], 3), ['regexCommand']); + // Nothing remotely similar: say nothing rather than guess. + assert.deepEqual(suggestParserRules('rexCommand', ['whereClause', 'sortCommand'], 3), []); +}); + +// --- report ------------------------------------------------------------------ + +test('the report groups by action, most urgent first', () => { + const drifts = [ + classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 1, + severities: ['error'], + backendRejected: true, + backendType: 'X', + backendReason: 'new wording', + }, + expectedBackend: { body: { error: { type: 'X', reason: 'old wording' } } }, + }) + ), + classifyDrift( + agreeingTrigger({ + version: '3.9.0', + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ), + ]; + const report = formatDriftReport(drifts); + assert.match(report, /2 finding\(s\) across 2 engine version\(s\)/); + assert.ok( + report.indexOf(REMEDIATIONS.VERSION_SCOPE_RULE) < report.indexOf(REMEDIATIONS.UPDATE_CONTRACT), + 'version scoping (a live false positive) must be listed before a stale-string re-pin' + ); + assert.match(report, /QUERY: union \[ source=t \]/); +}); + +test('an empty drift list reports agreement', () => { + assert.match(formatDriftReport([]), /No engine\/linter drift detected/); +}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs new file mode 100644 index 00000000000..7b3f6d174fd --- /dev/null +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -0,0 +1,554 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Multi-version aggregation for the PPL lint contract. + * + * The single-version workflow answers "do the OSD detectors and this engine + * agree?". This script answers the question that actually protects users: "does + * every default-ERROR rule still agree with EVERY supported engine version, and + * if not, what should the linter engineer change?" + * + * Inputs: one `--leg =` per engine version, where holds that + * leg's `target.json`, `backend-report.json`, `detector-report.json` and + * `ppl-grammar-bundle.json` (the same four files the single-version jobs already + * produce — this script adds no new producer). + * + * Output: a `drift-report.json` plus a markdown remediation report. Exits + * non-zero when any ENFORCED rule drifted on any version, so the check is red + * exactly when a shipped default-error rule disagrees with a supported engine. + * + * Usage: + * node scripts/ppl-lint/aggregate-versions.mjs \ + * --contracts integ-test/src/test/resources/ppl-lint/contracts \ + * --leg 3.6.0=legs/3.6.0 --leg 3.7.0=legs/3.7.0 --leg 3.8.0=legs/3.8.0 \ + * --out drift-report.json [--summary $GITHUB_STEP_SUMMARY] [--all-rules] + * + * By default only the manifest's `defaultError` set is enforced; `--all-rules` + * widens the report (still only enforcing `defaultError`) for nightly coverage. + */ + +import fs from 'fs'; +import path from 'path'; + +import { + classifyDrift, + classifyGrammarDrift, + formatDriftReport, + versionInAppliesTo, +} from './drift.mjs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-multiversion] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-multiversion] FATAL: ${message}`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { legs: [], contracts: '', out: 'drift-report.json', summary: '', allRules: false }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--leg') { + const raw = next(); + const eq = raw.indexOf('='); + if (eq <= 0) fatal(`--leg expects =, got "${raw}"`); + args.legs.push({ version: raw.slice(0, eq), dir: raw.slice(eq + 1) }); + } else if (arg === '--contracts') { + args.contracts = next(); + } else if (arg === '--out') { + args.out = next(); + } else if (arg === '--summary') { + args.summary = next(); + } else if (arg === '--all-rules') { + args.allRules = true; + } else { + fatal(`unknown argument "${arg}"`); + } + } + if (args.legs.length === 0) fatal('at least one --leg = is required'); + if (!args.contracts) fatal('--contracts is required'); + return args; +} + +function readJson(file, { optional = false } = {}) { + if (!fs.existsSync(file)) { + if (optional) return undefined; + fatal(`expected file not found: ${file}`); + } + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + if (optional) return undefined; + fatal(`could not parse ${file}: ${error.message}`); + } + return undefined; +} + +/** Load the contract corpus, keyed by ruleId, plus the manifest's enforced sets. */ +function loadContracts(dir) { + const manifest = readJson(path.join(dir, 'manifest.json')); + const specs = new Map(); + for (const name of manifest.contracts || []) { + const spec = readJson(path.join(dir, name)); + specs.set(spec.ruleId, { spec, file: name }); + } + // `defaultError` is the multi-version enforced set: every rule that ships + // enabled at error severity. Fall back to `enforced` for older manifests so + // this script still runs against an un-migrated corpus. + const enforcedFiles = new Set(manifest.defaultError || manifest.enforced || []); + const enforcedRules = new Set(); + for (const [ruleId, { file }] of specs) { + if (enforcedFiles.has(file)) enforcedRules.add(ruleId); + } + return { specs, enforcedRules, manifest }; +} + +/** + * Read one engine version's four artifacts. A leg whose backend never came up + * is fatal rather than skipped: silently dropping a version would turn a broken + * matrix into a green check, which is the failure mode this whole contract + * exists to prevent. + */ +function loadLeg({ version, dir }) { + const target = readJson(path.join(dir, 'target.json')); + const detector = readJson(path.join(dir, 'detector-report.json')); + const backendRaw = readJson(path.join(dir, 'backend-report.json')); + const bundle = readJson(path.join(dir, 'ppl-grammar-bundle.json'), { optional: true }); + + const backend = new Map(); + for (const entry of Array.isArray(backendRaw) ? backendRaw : []) { + backend.set(`${entry.ruleId}::${entry.queryName}`, entry); + } + + // The engine's self-reported version wins over the matrix label, so a matrix + // typo (asking for 3.7.0 and getting 3.8.0) cannot silently mislabel results. + const reported = target.engineVersion || ''; + if (reported && !reported.startsWith(version.split('-')[0])) { + log( + `WARN: leg "${version}" reported engineVersion "${reported}"; using the reported value for ` + + `version comparisons.` + ); + } + + return { + version: reported || version, + label: version, + dir, + grammarHash: target.grammarHash || '', + parserRuleNames: bundle && Array.isArray(bundle.parserRuleNames) ? bundle.parserRuleNames : undefined, + detector, + backend, + }; +} + +/** + * Compare the OSD catalog's default-error census (recorded by each detector leg) + * against the contracts this run knows about. Returns one entry per rule that + * ships enabled at error severity with no contract, or whose contract the + * manifest does not list under `defaultError`. + * + * Legs can disagree if they ran against different OSD checkouts, so the union is + * used: a rule that is default-error on ANY validated OSD ref must be accounted + * for. + */ +function auditDefaultErrorCensus(legs, specs, enforcedRules) { + const census = new Set(); + let sawCensus = false; + for (const leg of legs) { + const rules = leg.detector && leg.detector.defaultErrorRules; + if (!Array.isArray(rules)) continue; + sawCensus = true; + for (const ruleId of rules) census.add(ruleId); + } + if (!sawCensus) { + log( + "WARN: no detector leg reported a defaultErrorRules census, so the manifest's defaultError set " + + 'could not be cross-checked against the OSD catalog. Re-run with a detector build that emits it.' + ); + return []; + } + + const missing = []; + for (const ruleId of [...census].sort()) { + if (!specs.has(ruleId)) { + missing.push({ ruleId, reason: 'no contract file' }); + } else if (!enforcedRules.has(ruleId)) { + missing.push({ + ruleId, + reason: 'contract exists but is not listed under manifest.defaultError', + }); + } + } + return missing; +} + +/** + * Check an out-of-scope rule for the one drift that still matters there: the + * engine rejects a trigger query, but the rule's `appliesTo` excludes this + * version, so users on it see no diagnostic for a real error. Everything else + * about an out-of-scope rule is intentional silence. + * + * The trigger queries come from the spec's own `queries` map (there is no + * expectation to read on this path), and the backend observation from this leg's + * report; `classifyDrift` decides, so the "too narrow" wording stays in one place. + */ +function classifyOutOfScope({ spec, ruleId, leg, classify }) { + const found = []; + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + if ((queryDef.role || 'trigger') !== 'trigger') continue; + const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); + if (!backendEntry) continue; // this leg never ran the query + const observedBackend = backendEntry.observed || {}; + const detectorResult = (leg.detector.results || []).find( + (r) => r.ruleId === ruleId && r.queryName === queryName + ); + const drift = classify({ + ruleId, + version: leg.version, + queryName, + role: 'trigger', + query: queryDef.query.split('{{index}}').join(spec.index), + // Out of scope means the rule is expected to stay silent here. + expected: { detectorCount: 0 }, + observed: { + detectorCount: detectorResult ? detectorResult.actual : 0, + severities: detectorResult ? detectorResult.severities || [] : [], + backendRejected: !!backendEntry.rejected, + backendType: observedBackend.type, + backendReason: observedBackend.reason, + }, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + // Deliberately no parser-rule check here: a grammar that lacks the rule is + // expected on an engine the command predates. + }); + if (drift) found.push(drift); + } + return found; +} + +/** + * Pick the contract expectation that applies to a version, reusing the same + * "exactly one must match" rule as the two single-version halves. Returns + * undefined when the corpus does not cover this version — reported separately as + * a coverage hole, not as behavioral drift. + */ +function selectExpectation(spec, version, versionMatchesRange) { + const matches = (spec.expectations || []).filter((exp) => versionMatchesRange(exp.version, version)); + return matches.length === 1 ? matches[0] : undefined; +} + +/** Minimal semver-range test, kept byte-compatible with the other two halves. */ +function makeRangeMatcher() { + const parse = (v) => { + const m = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(v || '')); + return m ? [Number(m[1]), Number(m[2] || 0), Number(m[3] || 0)] : undefined; + }; + const cmp = (a, b) => { + for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + return 0; + }; + return (range, version) => { + if (!range || !String(range).trim()) return true; + const have = parse(version); + if (!have) return true; + for (const token of String(range).trim().split(/\s+/)) { + let op = '='; + let ver = token; + for (const candidate of ['>=', '<=', '>', '<', '=']) { + if (token.startsWith(candidate)) { + op = candidate; + ver = token.slice(candidate.length); + break; + } + } + const c = cmp(have, parse(ver) || [0, 0, 0]); + const ok = + (op === '>=' && c >= 0) || + (op === '<=' && c <= 0) || + (op === '>' && c > 0) || + (op === '<' && c < 0) || + (op === '=' && c === 0); + if (!ok) return false; + } + return true; + }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const versionMatchesRange = makeRangeMatcher(); + const { specs, enforcedRules, manifest } = loadContracts(args.contracts); + const legs = args.legs.map(loadLeg); + + log(`contracts=${specs.size} enforced(default-error)=${enforcedRules.size} legs=${legs.length}`); + for (const leg of legs) { + log( + ` leg ${leg.label}: engine=${leg.version} grammar=${(leg.grammarHash || '—').slice(0, 19)} ` + + `detectorResults=${(leg.detector.results || []).length} backendCases=${leg.backend.size}` + ); + } + + const drifts = []; + const coverageHoles = []; + const matrix = []; // one row per rule × version, for the summary table + + // A rule that ships enabled at error severity but has no contract file is + // invisible to this whole check. Compare the manifest's declared set against + // the census each detector leg recorded from the OSD catalog it linted with, so + // a new default-error rule cannot land unvalidated. + const missingContracts = auditDefaultErrorCensus(legs, specs, enforcedRules); + + for (const [ruleId, { spec, file }] of specs) { + const isEnforced = enforcedRules.has(ruleId); + if (!isEnforced && !args.allRules) continue; + + // A rule the catalog does not apply to an engine version ships nothing to + // users there, so it needs no expectation for it. Only a rule that IS in + // scope and has no expectation is a genuine hole. + const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; + + for (const leg of legs) { + const inScope = versionInAppliesTo(appliesTo, leg.version); + + // A parser rule that vanished from the grammar is one fact about this + // rule on this engine, not one per query — raise it once and move on, so + // the report shows the single edit to make instead of the same paragraph + // repeated for every case. + if (inScope) { + const grammarDrift = classifyGrammarDrift({ + ruleId, + version: leg.version, + requiredParserRules: spec.requiredParserRules, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + }); + if (grammarDrift) { + drifts.push({ ...grammarDrift, enforced: isEnforced, contractFile: file }); + matrix.push({ ruleId, version: leg.version, status: 'drift', drifts: 1 }); + continue; + } + } + + const expectation = selectExpectation(spec, leg.version, versionMatchesRange); + if (!expectation) { + if (!inScope) { + // Deliberately out of scope on this engine. Still run the classifier + // for the one case that matters — an engine that rejects a trigger the + // rule has been scoped away from (a missed diagnostic). + const outOfScopeDrifts = classifyOutOfScope({ + spec, + ruleId, + leg, + classify: classifyDrift, + }); + for (const drift of outOfScopeDrifts) { + drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + } + matrix.push({ + ruleId, + version: leg.version, + status: outOfScopeDrifts.length > 0 ? 'drift' : 'out-of-scope', + drifts: outOfScopeDrifts.length, + }); + continue; + } + // In scope on this engine but nothing pins its behavior there. + coverageHoles.push({ ruleId, file, version: leg.version, enforced: isEnforced }); + matrix.push({ ruleId, version: leg.version, status: 'uncovered', drifts: 0 }); + continue; + } + + let ruleDrifts = 0; + for (const [queryName, expected] of Object.entries(expectation.queries || {})) { + const queryDef = (spec.queries || {})[queryName]; + if (!queryDef) continue; // the single-version halves already fail on this + const query = queryDef.query.split('{{index}}').join(spec.index); + const role = queryDef.role || 'trigger'; + + const detectorResult = (leg.detector.results || []).find( + (r) => r.ruleId === ruleId && r.queryName === queryName + ); + const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); + const observedBackend = backendEntry && backendEntry.observed; + + const drift = classifyDrift({ + ruleId, + version: leg.version, + queryName, + role, + query, + expected: { + detectorCount: expected.detectorCount, + severity: expected.severity, + backendKind: expected.backend && expected.backend.kind, + }, + observed: { + detectorCount: detectorResult ? detectorResult.actual : 0, + severities: detectorResult ? detectorResult.severities || [] : [], + backendRejected: backendEntry ? !!backendEntry.rejected : undefined, + backendType: observedBackend ? observedBackend.type : undefined, + backendReason: observedBackend ? observedBackend.reason : undefined, + }, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + requiredParserRules: spec.requiredParserRules, + expectedBackend: expected.backend, + }); + + if (drift) { + drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + ruleDrifts++; + } + } + matrix.push({ + ruleId, + version: leg.version, + status: ruleDrifts === 0 ? 'agree' : 'drift', + drifts: ruleDrifts, + }); + } + } + + const enforcedDrifts = drifts.filter((d) => d.enforced); + const enforcedHoles = coverageHoles.filter((h) => h.enforced); + + const report = { + schemaVersion: 1, + legs: legs.map((l) => ({ + label: l.label, + engineVersion: l.version, + grammarHash: l.grammarHash, + })), + enforcedRules: [...enforcedRules].sort(), + missingContracts, + manifestDescription: manifest.description || '', + matrix, + drifts, + coverageHoles, + result: { + driftCount: drifts.length, + enforcedDriftCount: enforcedDrifts.length, + enforcedCoverageHoles: enforcedHoles.length, + missingContractCount: missingContracts.length, + passed: + enforcedDrifts.length === 0 && + enforcedHoles.length === 0 && + missingContracts.length === 0, + }, + }; + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + log(`wrote ${args.out}`); + + const markdown = renderMarkdown(report, drifts, coverageHoles, legs); + // eslint-disable-next-line no-console + console.log(markdown); + if (args.summary) { + try { + fs.appendFileSync(args.summary, markdown + '\n'); + } catch (error) { + log(`WARN: could not write summary to ${args.summary}: ${error.message}`); + } + } + + if (!report.result.passed) { + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-multiversion] FAIL: ${enforcedDrifts.length} drift(s), ` + + `${enforcedHoles.length} coverage hole(s) and ${missingContracts.length} unvalidated ` + + `default-error rule(s).` + ); + process.exit(1); + } + log( + `PASS: every default-error rule agrees with all ${legs.length} engine version(s)` + + (drifts.length > 0 ? ` (${drifts.length} non-enforced finding(s) reported)` : '') + + '.' + ); +} + +/** Rule × version agreement matrix followed by the grouped remediation report. */ +function renderMarkdown(report, drifts, coverageHoles, legs) { + const lines = []; + lines.push('## PPL lint multi-version validation'); + lines.push(''); + lines.push( + `Engine versions: ${legs.map((l) => `\`${l.version}\``).join(', ')} — ` + + `**${report.result.passed ? 'PASS' : 'FAIL'}** ` + + `(${report.result.enforcedDriftCount} enforced drift(s), ` + + `${report.result.enforcedCoverageHoles} coverage hole(s))` + ); + lines.push(''); + + const versions = legs.map((l) => l.version); + const rules = [...new Set(report.matrix.map((m) => m.ruleId))].sort(); + lines.push(`| Rule | ${versions.map((v) => `\`${v}\``).join(' | ')} |`); + lines.push(`| ---- | ${versions.map(() => '----').join(' | ')} |`); + const cell = { + agree: 'agree', + drift: 'DRIFT', + uncovered: 'not covered', + 'out-of-scope': 'n/a (out of scope)', + }; + for (const ruleId of rules) { + const cells = versions.map((version) => { + const row = report.matrix.find((m) => m.ruleId === ruleId && m.version === version); + if (!row) return '—'; + return row.status === 'drift' ? `**DRIFT** (${row.drifts})` : cell[row.status]; + }); + lines.push(`| \`${ruleId}\` | ${cells.join(' | ')} |`); + } + lines.push(''); + + if ((report.missingContracts || []).length > 0) { + lines.push('### Unvalidated default-error rules'); + lines.push(''); + for (const entry of report.missingContracts) { + lines.push( + `- \`${entry.ruleId}\` ships enabled at error severity but ${entry.reason}, so no engine ` + + `version validates it. FIX: add \`${entry.ruleId}.spec.json\` under ` + + `integ-test/src/test/resources/ppl-lint/contracts/ with a trigger + control query and list ` + + `it in manifest.json under \`defaultError\`. If the rule should not be default-error, lower ` + + `its severity or disable it in packages/osd-monaco/src/ppl/lint/rules_catalog.json.` + ); + } + lines.push(''); + } + + if (coverageHoles.length > 0) { + lines.push('### Coverage holes'); + lines.push(''); + for (const hole of coverageHoles) { + lines.push( + `- \`${hole.ruleId}\` has no expectation matching engine \`${hole.version}\`` + + `${hole.enforced ? ' (ENFORCED — this rule ships to users on that engine unpinned)' : ''}. ` + + `FIX (${hole.file}): add an \`expectations[]\` entry whose \`version\` range covers ` + + `\`${hole.version}\`, or narrow the rule's \`appliesTo\` so it does not apply there.` + ); + } + lines.push(''); + } + + lines.push('### Remediation'); + lines.push(''); + lines.push('```'); + lines.push(formatDriftReport(drifts)); + lines.push('```'); + return lines.join('\n'); +} + +main(); diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs new file mode 100644 index 00000000000..b343778ac3e --- /dev/null +++ b/scripts/ppl-lint/drift.mjs @@ -0,0 +1,556 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Drift classification for the multi-version PPL lint contract. + * + * The contract runs every default-error lint rule against SEVERAL engine + * versions. Detecting that a rule disagrees with an engine is only half the + * job — a bare "expected 1 diagnostic, got 0" tells an engineer that something + * moved but not what to do about it. This module turns one observed + * (expected vs detector vs backend) triple into: + * + * 1. a drift CLASS — which of the known ways engine/linter can diverge; and + * 2. a REMEDIATION — the concrete linter-side action, one of: + * disable-rule the engine no longer has the behavior; stop shipping + * the diagnostic (or scope it to older versions) + * version-scope-rule the behavior is version-bounded; fix appliesTo + * update-detector the detector's own logic/anchor is now wrong + * update-contract the linter is right; the pinned expectation is stale + * + * The classifier is deliberately pure and side-effect free so it can be unit + * tested without a cluster (see __tests__/drift.test.mjs). The runner supplies + * observations; this module decides nothing about how they were gathered. + * + * Naming: a "trigger" query is one the rule is supposed to flag; a "control" is + * a near-identical valid query it must stay silent on. `role` distinguishes them. + */ + +/** Every drift class this module can emit, with a stable one-line meaning. */ +export const DRIFT_CLASSES = { + GRAMMAR_RULE_MISSING: 'grammar-rule-missing', + ENGINE_RELAXED: 'engine-relaxed', + ENGINE_TIGHTENED: 'engine-tightened', + ENGINE_MESSAGE_CHANGED: 'engine-message-changed', + DETECTOR_SILENT: 'detector-silent', + DETECTOR_NOISY: 'detector-noisy', + VERSION_SCOPE_TOO_NARROW: 'version-scope-too-narrow', + SEVERITY_MISMATCH: 'severity-mismatch', +}; + +/** Remediation actions, phrased as what the linter engineer changes. */ +export const REMEDIATIONS = { + DISABLE_RULE: 'disable-rule', + VERSION_SCOPE_RULE: 'version-scope-rule', + UPDATE_DETECTOR: 'update-detector', + UPDATE_CONTRACT: 'update-contract', +}; + +/** OSD paths an engineer edits, kept in one place so a move is a one-line fix. */ +const OSD_PATHS = { + catalog: 'packages/osd-monaco/src/ppl/lint/rules_catalog.json', + ruleDir: 'packages/osd-monaco/src/ppl/lint/rules/', + ruleIndex: 'packages/osd-monaco/src/ppl/lint/rule_index.ts', +}; + +/** + * Path of the detector implementation for a rule. + * + * Most rules follow the snake_case-of-the-id convention, but not all: the + * catalog id `unsupported-window-function-in-eventstats` lives in + * `unsupported_window_function.ts`. A remediation that names a file the engineer + * cannot open is worse than one that names a directory, so a contract may pin the + * real path via `detectorPath` and we fall back to the convention otherwise. + */ +function detectorFile(ruleId, detectorPath) { + if (detectorPath) { + return detectorPath; + } + return `${OSD_PATHS.ruleDir}${String(ruleId).replace(/-/g, '_')}.ts`; +} + +/** + * Cheap edit-distance, used only to suggest "did the grammar rename X to Y?". + * Bounded by the shorter string, so it is O(n*m) on short identifiers. + */ +function editDistance(a, b) { + const m = a.length; + const n = b.length; + if (m === 0 || n === 0) return Math.max(m, n); + let prev = Array.from({ length: n + 1 }, (_, j) => j); + for (let i = 1; i <= m; i++) { + const row = [i]; + for (let j = 1; j <= n; j++) { + const cost = a[i - 1] === b[j - 1] ? 0 : 1; + row[j] = Math.min(prev[j] + 1, row[j - 1] + 1, prev[j - 1] + cost); + } + prev = row; + } + return prev[n]; +} + +/** Split a camelCase parser rule name into lower-case tokens. */ +function camelTokens(name) { + return String(name) + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); +} + +/** + * Best candidates for a parser rule that vanished from the candidate grammar. + * + * Ranked by how a real ANTLR rename tends to look, strongest signal first: + * 0. containment `unionCommand` -> `unionCommandNew` + * 1. same leading token `unionCommand` -> `unionStatement` + * 2. near spelling `rexCommand` -> `regexCommand` + * + * The leading token carries the identity of the rule; the trailing one is + * usually a generic suffix (`Command`, `Clause`, `Expression`) shared by most of + * the grammar, so matching on it alone would suggest dozens of unrelated rules. + * That is why only the FIRST token counts, and why an unrelated rule returns + * nothing rather than a plausible-looking wrong guess. + */ +export function suggestParserRules(missingRule, availableRules, limit = 3) { + const missing = String(missingRule); + const lower = missing.toLowerCase(); + const missingHead = camelTokens(missing)[0]; + const scored = []; + for (const candidate of availableRules) { + const cl = String(candidate).toLowerCase(); + let score; + if (cl.includes(lower) || lower.includes(cl)) { + score = 0; // containment: strongest signal of a rename + } else if (missingHead && camelTokens(candidate)[0] === missingHead) { + score = 1; // same subject, renamed suffix + } else { + // Only very near spellings survive this tier. A looser budget scaled to + // name length lets long names match unrelated same-suffix rules + // (`unionCommand` vs `binCommand` differ by 4 edits but are unrelated), so + // the cap is absolute: typo-or-insertion distance, nothing more. + const distance = editDistance(lower, cl); + if (distance > 2) continue; + score = 1 + distance; + } + scored.push({ candidate, score }); + } + scored.sort((a, b) => a.score - b.score || String(a.candidate).localeCompare(String(b.candidate))); + return scored.slice(0, limit).map((s) => s.candidate); +} + +/** Parse "3.8.0-SNAPSHOT" / "3.7" into [major, minor, patch]; undefined if unparseable. */ +export function parseVersion(value) { + if (!value) return undefined; + const match = /^(\d+)(?:\.(\d+))?(?:\.(\d+))?/.exec(String(value)); + if (!match) return undefined; + return [Number(match[1]), Number(match[2] || 0), Number(match[3] || 0)]; +} + +export function compareVersion(a, b) { + for (let i = 0; i < 3; i++) { + if (a[i] !== b[i]) return a[i] < b[i] ? -1 : 1; + } + return 0; +} + +/** + * True when `version` falls inside the catalog's `appliesTo` window. An absent + * bound is open-ended, and an unparseable version is treated as in-range so an + * unrecognized engine build never silently drops coverage. + */ +export function versionInAppliesTo(appliesTo, version) { + const have = parseVersion(version); + if (!have) return true; + const min = parseVersion(appliesTo && appliesTo.minVersion); + const max = parseVersion(appliesTo && appliesTo.maxVersion); + if (min && compareVersion(have, min) < 0) return false; + // maxVersion is treated as inclusive, matching the OSD catalog's own reading. + if (max && compareVersion(have, max) > 0) return false; + return true; +} + +/** Human-readable one-liner for the observed pair, reused across messages. */ +function describeObservation(observed) { + const detector = observed.detectorCount > 0 ? `flagged (${observed.detectorCount})` : 'silent'; + let backend = 'accepted'; + if (observed.backendRejected) { + const type = observed.backendType ? ` ${observed.backendType}` : ''; + backend = `rejected${type}`; + } else if (observed.backendRejected === undefined) { + backend = 'not observed'; + } + return `detector ${detector}, engine ${backend}`; +} + +/** + * Report a parser rule the detector walks that the candidate grammar no longer + * defines. Exported so a caller can raise it ONCE per rule/version — the fact is + * a property of the grammar, not of any single query, and repeating it per query + * buries the one edit an engineer has to make. `classifyDrift` still calls it so + * a caller that does not hoist the check keeps the diagnosis. + */ +export function classifyGrammarDrift({ + ruleId, + version, + requiredParserRules, + parserRuleNames, + observed = {}, + queryName, + role = 'trigger', + query, + detectorPath, +}) { + if (!Array.isArray(requiredParserRules) || !Array.isArray(parserRuleNames)) { + return null; + } + const available = new Set(parserRuleNames); + const missing = requiredParserRules.filter((rule) => !available.has(rule)); + if (missing.length === 0) { + return null; + } + const missingList = missing.map((r) => `"${r}"`).join(', '); + const suggestions = [...new Set(missing.flatMap((rule) => suggestParserRules(rule, parserRuleNames)))]; + const at = queryName ? ` [${queryName}]` : ''; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + driftClass: DRIFT_CLASSES.GRAMMAR_RULE_MISSING, + evidence: + `${ruleId} @ ${version}${at}: the candidate grammar has no parser rule(s) ${missingList}, ` + + `which this rule's detector walks.` + + (observed && observed.detectorCount !== undefined ? ` ${describeObservation(observed)}.` : ''), + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine grammar renamed or removed ${missingList}. ` + + (suggestions.length > 0 + ? `Closest rule(s) now in the grammar: ${suggestions.map((r) => `"${r}"`).join(', ')}. ` + : '') + + `Re-anchor the detector (and ${OSD_PATHS.ruleIndex} if the name is listed there) onto the ` + + `current rule name, then update this contract's requiredParserRules. If the command itself ` + + `is gone from the engine, disable the rule instead.`, + }, + }; +} + +/** + * Classify one query's outcome on one engine version. + * + * Returns `null` when the detector, the engine and the pinned expectation all + * agree — the overwhelmingly common case. Otherwise returns a single drift + * object; checks run most-specific-first so the reported cause is the root one + * (a renamed grammar rule explains a silent detector, not the other way round). + * + * @param {object} input + * @param {string} input.ruleId + * @param {string} input.version engine version under test, e.g. "3.7.0" + * @param {string} input.queryName + * @param {string} input.role 'trigger' | 'control' + * @param {string} input.query the query as sent to both halves + * @param {object} input.expected { detectorCount, severity, backendKind } + * @param {object} input.observed { detectorCount, severities, backendRejected, backendType, backendReason } + * @param {object} [input.wiring] OSD catalog entry (appliesTo, runtimeOnly, ...) + * @param {string[]} [input.parserRuleNames] candidate grammar's parser rule names + * @param {string[]} [input.requiredParserRules] grammar rules the detector walks + * @param {object} [input.expectedBackend] contract's pinned rejection body + */ +export function classifyDrift(input) { + const { + ruleId, + version, + queryName, + role = 'trigger', + query, + expected = {}, + observed = {}, + wiring, + parserRuleNames, + requiredParserRules, + expectedBackend, + detectorPath, + } = input; + + const detectorFlagged = (observed.detectorCount || 0) > 0; + const expectFlagged = (expected.detectorCount || 0) > 0; + const backendRejected = observed.backendRejected; + const where = `${ruleId} @ ${version} [${queryName}]`; + const base = { ruleId, version, queryName, role, query, driftVersion: version }; + + // --- 1. Did the grammar move out from under the detector? ------------------- + // A detector that walks a parser rule the candidate grammar no longer defines + // cannot fire at all. This is the root cause of an otherwise baffling silent + // detector, so it is checked before any behavioral comparison. + const grammarDrift = classifyGrammarDrift({ + ruleId, + version, + requiredParserRules, + parserRuleNames, + observed, + queryName, + role, + query, + detectorPath, + }); + if (grammarDrift) { + return grammarDrift; + } + + // --- 2. Is the rule even in scope for this engine version? ------------------ + // A rule whose appliesTo excludes this version is intentionally inert here. + // That is only correct if the engine also does not exhibit the behavior; if the + // engine rejects the trigger, the version window is too narrow and users on + // this version get no diagnostic. + const inScope = versionInAppliesTo(wiring && wiring.appliesTo, version); + if (!inScope) { + if (role === 'trigger' && backendRejected === true) { + return { + ...base, + driftClass: DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW, + evidence: + `${where}: engine ${version} rejects this trigger, but the rule's appliesTo ` + + `(${JSON.stringify((wiring && wiring.appliesTo) || {})}) excludes ${version}, so no diagnostic ` + + `is shown to users on that version.`, + remediation: { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `Widen "${ruleId}".appliesTo to include ${version} (lower minVersion / raise maxVersion) so the ` + + `diagnostic reaches users on engines that actually reject the query.`, + }, + }; + } + // Out of scope and the engine agrees it is a non-issue: nothing to report. + return null; + } + + // --- 3. Behavioral flips: the engine changed its verdict -------------------- + const expectRejection = expected.backendKind === 'rejection'; + + // 3a. The engine now ACCEPTS what the contract pinned as a rejection. Any + // diagnostic the linter still emits is a false positive shipped to users — + // the single most damaging drift, so it is reported even when the detector + // count happens to match the stale expectation. + if (role === 'trigger' && expectRejection && backendRejected === false) { + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS a query the contract pinned as rejected ` + + `(${describeObservation(observed)}). The engine gained support for this construct.`, + remediation: detectorFlagged + ? { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `"${ruleId}" is now a FALSE POSITIVE on ${version}. Bound it to the versions that still ` + + `reject: set appliesTo.maxVersion just below ${version}. If no supported engine rejects it ` + + `any more, set "enabled": false (disable-rule) and drop the detector. Then re-pin this ` + + `contract's ${version} expectation to detectorCount 0 / backend.kind "result-shape".`, + } + : { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already stays silent on ${version}, so no linter change is needed. Re-pin the ` + + `${version} expectation to detectorCount 0 / backend.kind "result-shape" to record the ` + + `engine's new behavior.`, + }, + }; + } + + // 3b. The engine now REJECTS what the contract pinned as valid. A control that + // started failing means the linter is silently missing a real error. + if (!expectRejection && backendRejected === true) { + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_TIGHTENED, + evidence: + `${where}: engine ${version} now REJECTS a query the contract pinned as valid ` + + `(${observed.backendType || 'error'}: ${observed.backendReason || 'no reason'}). ` + + `${describeObservation(observed)}.`, + remediation: detectorFlagged + ? { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already flags this, so the linter is correct and only the pinned expectation ` + + `is stale. Re-pin the ${version} expectation to backend.kind "rejection" with the observed ` + + `error.type/reason, and pick a genuinely valid query for the control.`, + } + : { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine rejects this but the linter is silent — a false NEGATIVE on ${version}. Extend ` + + `the detector to cover this shape (or, if the rejection belongs to a different rule, add a ` + + `contract case under that rule). Then re-pin this expectation.`, + }, + }; + } + + // --- 4. Same verdict, different wording ------------------------------------ + // The engine still rejects, but the error type/reason moved. The detector is + // still right; the pinned body — and any detector text that quotes the engine + // wording — is stale. Worth flagging because linter messages and quick-fix + // copy are written against these strings. + if (backendRejected === true && expectRejection && expectedBackend) { + const expectedError = (expectedBackend.body && expectedBackend.body.error) || {}; + const typeChanged = + expectedError.type !== undefined && + observed.backendType !== undefined && + expectedError.type !== observed.backendType; + const reasonChanged = + expectedError.reason !== undefined && + observed.backendReason !== undefined && + expectedError.reason !== observed.backendReason; + if (typeChanged || reasonChanged) { + const parts = []; + if (typeChanged) parts.push(`error.type "${expectedError.type}" -> "${observed.backendType}"`); + if (reasonChanged) + parts.push(`error.reason "${expectedError.reason}" -> "${observed.backendReason}"`); + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_MESSAGE_CHANGED, + evidence: + `${where}: engine ${version} still rejects the query but reworded the failure — ${parts.join('; ')}.`, + remediation: { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector's verdict is unaffected, so no rule change is required. Update the ${version} ` + + `expectation's backend.body to the observed wording. Also check whether "${ruleId}"'s message ` + + `or quick-fix copy in ${detectorFile(ruleId, detectorPath)} quotes the old engine wording.`, + }, + }; + } + } + + // --- 5. Detector-only disagreements ---------------------------------------- + // The engine behaved as pinned, so any mismatch is on the linter side. + if (expectFlagged && !detectorFlagged) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_SILENT, + evidence: + `${where}: expected ${expected.detectorCount} diagnostic(s) but the detector produced none, ` + + `while the engine behaved as pinned (${describeObservation(observed)}).`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine still exhibits the behavior, so the rule is still wanted — the detector regressed. ` + + `Check, in order: (1) appliesTo/minVersion vs engine ${version}; (2) runtimeOnly — a runtimeOnly ` + + `rule only fires when the lint context's grammarSurface is "runtime-bundle"; (3) required lint ` + + `context (fields/typeMap) that the detector self-suppresses without; (4) the detector's own ` + + `traversal. Do NOT re-pin the expectation to 0 — that would hide a false negative.`, + }, + }; + } + + if (!expectFlagged && detectorFlagged) { + const engineAgrees = backendRejected === true; + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_NOISY, + evidence: + `${where}: expected no diagnostic but the detector emitted ${observed.detectorCount} ` + + `(${describeObservation(observed)}).`, + remediation: engineAgrees + ? { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The engine rejects this query too, so the diagnostic is arguably correct and the ` + + `expectation is what is wrong. Re-pin the ${version} expectation, or choose a control query ` + + `the engine actually accepts.`, + } + : { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `The engine ACCEPTS this query, so the diagnostic is a false positive on ${version}. Narrow ` + + `the detector so it stops matching this shape; if the whole rule no longer applies to any ` + + `supported engine, disable it in ${OSD_PATHS.catalog}.`, + }, + }; + } + + // --- 6. Right verdict, wrong severity -------------------------------------- + if ( + expected.severity && + detectorFlagged && + Array.isArray(observed.severities) && + observed.severities.length > 0 && + !observed.severities.every((s) => s === expected.severity) + ) { + return { + ...base, + driftClass: DRIFT_CLASSES.SEVERITY_MISMATCH, + evidence: + `${where}: expected severity "${expected.severity}" but the detector emitted ` + + `${JSON.stringify(observed.severities)}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: OSD_PATHS.catalog, + detail: + `Restore "${ruleId}".severity to "${expected.severity}" in the catalog, or — if the downgrade was ` + + `deliberate — re-pin this contract and note that the rule left the enforced default-error set.`, + }, + }; + } + + return null; +} + +/** + * Render drifts as the PR-facing remediation report. Grouped by remediation + * action so the reader sees the decision first ("two rules need version + * scoping") rather than a flat wall of query failures. + */ +export function formatDriftReport(drifts) { + if (drifts.length === 0) { + return 'No engine/linter drift detected.'; + } + const byAction = new Map(); + for (const drift of drifts) { + const action = drift.remediation.action; + if (!byAction.has(action)) byAction.set(action, []); + byAction.get(action).push(drift); + } + + const lines = [ + `PPL lint drift: ${drifts.length} finding(s) across ${new Set(drifts.map((d) => d.version)).size} engine version(s).`, + '', + ]; + // Most urgent action first: a false positive already reaching users outranks a + // stale pinned string. + const order = [ + REMEDIATIONS.DISABLE_RULE, + REMEDIATIONS.VERSION_SCOPE_RULE, + REMEDIATIONS.UPDATE_DETECTOR, + REMEDIATIONS.UPDATE_CONTRACT, + ]; + for (const action of order) { + const group = byAction.get(action); + if (!group || group.length === 0) continue; + lines.push(`## ${action} (${group.length})`); + for (const drift of group) { + lines.push(`- [${drift.driftClass}] ${drift.evidence}`); + lines.push(` FIX (${drift.remediation.target}): ${drift.remediation.detail}`); + // A rule-level finding (e.g. a grammar rename) has no single query behind it. + if (drift.query) { + lines.push(` QUERY: ${drift.query}`); + } + } + lines.push(''); + } + return lines.join('\n'); +} diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 8bd1ff5bad1..e45b4a48b04 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -419,6 +419,16 @@ function main() { engineVersion, grammarHash: target.grammarHash || '', differential: !!backendReport, + // Census of the rules that ship enabled at ERROR severity, read from the OSD + // catalog this run linted with. The multi-version aggregator enforces its + // `defaultError` manifest set against this list, so a rule that becomes + // default-error in OSD without a contract file cannot slip through + // unvalidated — and the aggregator does not need its own OSD checkout to + // notice (design: default-error is the set users cannot opt out of). + defaultErrorRules: catalog + .filter((rule) => rule.enabled && rule.severity === 'error') + .map((rule) => rule.id) + .sort(), results: [], }; From da0fa6d6a49d3c857cf66cc07175241c852691e7 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 16:33:14 -0700 Subject: [PATCH 03/39] fix(ci): stop the multi-version lint check from passing vacuously Self-review of the multi-version drift check found five ways it could either pass while a rule had actually drifted, or hand an engineer advice that would make things worse. All five are fixed with regression tests. 1. A transport failure was read as engine ACCEPTANCE. The IT marks an unanswered query outcome:"error" and writes no `rejected` field; the aggregator coerced that absence to false. A socket timeout therefore looked like an engine that had gained support for the construct, and the report told the engineer the rule was "now a FALSE POSITIVE ... set enabled:false and drop the detector". Observations now distinguish "engine accepted" from "no verdict received", and an uncomparable case is not classified at all -- a dead leg can no longer manufacture linter advice. 2. A reworded engine message masked a detector that had gone silent. The engine-message-changed branch returned before the detector-silent check without verifying the detector still agreed, reporting "the detector's verdict is unaffected, so no rule change is required". Re-pinning the string would have turned the check green over a rule that no longer fired. It now requires the detector count to match, so when both moved the silent detector is reported instead. 3. A rule whose every case was uncomparable reported "agree". "agree" now means "we compared something and it matched"; otherwise the pair is reported inconclusive and FAILS, with advice to check that leg's logs and re-run rather than to edit anything. 4. A dead observe job silently shrank the matrix. The aggregate step dropped legs with no report and then printed "agrees with all N versions" for the survivors. It now requires every planned version to have produced a leg. 5. Two narrower cases: a detector that fires on a version its appliesTo excludes was unreportable (reachable in production, since OSD's version filter runs a rule when the cluster version is unknown), and selectExpectation was missing the engine:"calcite" filter both single-version halves apply. Also: a fixture index that fails to seed on an older engine is now recorded, so its contracts report as unusable instead of turning IndexNotFoundException into a fake engine verdict -- which would otherwise have advised pinning the contract to that exception. Verified: 43 tests green (up from 35); the healthy 7-rule x 3-version matrix still passes and the five-injected-drift scenario still yields five correct remediations. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 20 +++ .../remote/PplLintRuleValidationIT.java | 71 ++++++++- scripts/ppl-lint/README.md | 16 +- .../__tests__/aggregate-versions.test.mjs | 110 +++++++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 52 ++++++ scripts/ppl-lint/aggregate-versions.mjs | 149 +++++++++++++++--- scripts/ppl-lint/drift.mjs | 46 +++++- 7 files changed, 436 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index f4ad09b6a82..e9ee6849111 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -397,19 +397,39 @@ jobs: # contracts, then print the remediation report. - name: Aggregate drift across engine versions id: aggregate + env: + RELEASED: ${{ needs.plan.outputs.released }} run: | set -euo pipefail shopt -s nullglob args=() + present=() for leg in "$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*; do [ -f "$leg/detector-report.json" ] || continue version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') args+=(--leg "$version=$leg") + present+=("$version") done if [ ${#args[@]} -eq 0 ]; then echo "::error::no complete legs to aggregate." exit 1 fi + # Every version the plan asked for must have produced a leg. Aggregating + # only the survivors would report "PASS: agrees with all N versions" over + # a matrix that silently lost one — the exact vacuous pass this workflow + # exists to prevent. A dead leg is a failure, not a smaller matrix. + missing=() + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") pr-build; do + found=no + for have in "${present[@]}"; do + [ "$have" = "$want" ] && found=yes && break + done + [ "$found" = yes ] || missing+=("$want") + done + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::planned engine leg(s) produced no report: ${missing[*]}. Check those observe jobs; the matrix is incomplete so its result would be misleading." + exit 1 + fi node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ --out "$GITHUB_WORKSPACE/drift-report.json" \ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index ae74fdd2515..065bc4e78eb 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -94,6 +94,13 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private int[] clusterVersion; private String engineVersionRaw; + /** + * Index fixtures that could not be created on this engine (observe-only mode only). Contracts + * that need one are reported as {@code outcome: "error"} instead of as engine behavior, because + * an IndexNotFoundException from a missing fixture is not a verdict about the query. + */ + private final Set unseededIndices = new LinkedHashSet<>(); + @Override public void init() throws Exception { super.init(); @@ -108,9 +115,16 @@ public void init() throws Exception { } // In the multi-version matrix an older engine may not support a field type // a fixture uses (a mapping that only exists in a later release). Losing - // that one index must not abort the whole leg — the contracts that need it - // will surface as their own observations, while every other rule is still + // that one index must not abort the whole leg — every other rule is still // validated against this engine. + // + // But it must not be silent either: without the index, every query against + // it fails with IndexNotFoundException, which looks exactly like a real + // engine verdict. Left unmarked, the drift report would advise pinning the + // contract to IndexNotFoundException, or "extending the detector" for a + // control the engine only rejected because its data was missing. Record the + // failure so those cases are reported as unusable rather than as behavior. + unseededIndices.add(indexEnum); System.err.println( "[ppl-lint] could not seed index " + indexEnum + " on this engine: " + e.getMessage()); } @@ -154,6 +168,16 @@ private void runContract( JSONObject fixture = contract.optJSONObject("backendFixture"); boolean calciteOn = fixtureCalciteEnabled(fixture); + // A contract whose fixture index never got created cannot produce a meaningful + // observation: every query would fail with IndexNotFoundException regardless of + // the rule. Report each case as an error so the aggregator counts it as + // inconclusive rather than as the engine's verdict. + String missingIndex = missingFixtureIndex(fixture); + if (missingIndex != null) { + recordUnusableContract(ruleId, index, queries, report, missingIndex); + return; + } + List applied = applyClusterSettings(fixture); try { JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); @@ -209,6 +233,49 @@ private void runContract( } } + /** + * The first index fixture this contract needs that failed to seed, or null when every index it + * declares is present. Only ever non-null in observe-only mode, where a seeding failure is + * tolerated instead of aborting the leg. + */ + private String missingFixtureIndex(JSONObject fixture) { + if (unseededIndices.isEmpty() || fixture == null) { + return null; + } + JSONArray declared = fixture.optJSONArray("indices"); + if (declared == null) { + return unseededIndices.contains("ACCOUNT") ? "ACCOUNT" : null; + } + for (int i = 0; i < declared.length(); i++) { + String name = declared.getString(i); + if (unseededIndices.contains(name)) { + return name; + } + } + return null; + } + + /** + * Record every case of a contract whose fixture index is missing as {@code outcome: "error"}, so + * the multi-version aggregator treats them as inconclusive. Writing nothing at all would be + * worse: absent rows are indistinguishable from a detector that never ran. + */ + private void recordUnusableContract( + String ruleId, String index, JSONObject queries, JSONArray report, String missingIndex) { + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + report.put( + reportEntry(ruleId, queryName, role, query, "observe-only") + .put("outcome", "error") + .put( + "error", + "fixture index " + missingIndex + " could not be created on this engine")); + log(ruleId, queryName, "SKIPPED (fixture index " + missingIndex + " unavailable)"); + } + } + /** * Observe-only helper: run every query a contract declares and record what the engine actually * did, without comparing against any expectation. Used when this engine version has no matching diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index a78b1f5cdcc..76b3d56a0e0 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -240,14 +240,26 @@ renamed or removed — the finding names the closest current rule names), (the engine rejects but the rule is scoped away from that version, so users see no diagnostic), and `severity-mismatch`. -Two guards keep the check from passing vacuously: +Four guards keep the check from passing vacuously. Each exists because "we could +not check" must never render as "it is fine": - A rule that is default-error in OSD's catalog but has no contract file fails the run. The detector runner records the catalog's default-error census in `detector-report.json`, and the aggregate step compares it against `manifest.defaultError` — so a new error rule cannot land unvalidated. - A leg whose artifacts are missing is a hard failure, never a silently dropped - version. + version. The aggregate step also checks that every version the plan asked for + produced a report, so a dead observe job cannot shrink the matrix into a green + "agrees with all N versions". +- A case with no engine verdict (a transport failure, recorded by the IT as + `outcome: "error"`) is **not** read as acceptance. Coercing it would report a + timeout as an engine that now accepts the query — and advise disabling a + perfectly good rule. Likewise, a contract whose fixture index failed to seed is + reported as unusable rather than as a stream of `IndexNotFoundException` + verdicts. +- A rule whose every case was uncomparable is reported `inconclusive` and **fails** + — it proved nothing. Inconclusive findings say "check that leg's logs and re-run", + never "edit the rule", because the linter is not what went wrong. A rule that is out of scope on an engine (`appliesTo` excludes it) and that the engine also accepts is reported as `n/a (out of scope)`, not as drift — that is diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 8f1bd7493a5..76e7b558181 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -341,6 +341,116 @@ test('a legacy detector report without a census warns instead of failing', () => assert.match(stdout, /no detector leg reported a defaultErrorRules census/); }); +// --- "we don't know" must never render as "it's fine" ------------------------- + +/** Write a leg where a named query produced no engine verdict (transport failure). */ +function writeLegWithTransportError({ version, erroredQuery, cases }) { + const dir = writeLeg({ version, cases }); + const file = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(file, 'utf8')).map((entry) => + entry.queryName === erroredQuery + ? // Exactly what the IT writes on a transport failure: an `error` outcome and + // NO `rejected` field, because no verdict was ever received. + { ruleId: entry.ruleId, queryName: entry.queryName, role: entry.role, outcome: 'error', error: 'connect timeout' } + : { ...entry, outcome: 'observed' } + ); + fs.writeFileSync(file, JSON.stringify(backend)); + return dir; +} + +test('a transport error is not read as engine acceptance', () => { + // Regression: coercing a missing `rejected` to false made a timeout look like an + // engine that now ACCEPTS the trigger, and advised disabling a healthy rule. + const legs = { + '3.7.0': writeLegWithTransportError({ + version: '3.7.0', + erroredQuery: 'trigger', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }), + }; + const { report, stdout } = run({ contracts: writeContracts(), legs }); + assert.equal( + report.drifts.filter((d) => d.driftClass === 'engine-relaxed').length, + 0, + 'a timeout must never be reported as the engine relaxing' + ); + assert.ok( + !/FALSE POSITIVE/.test(stdout), + 'a timeout must never advise disabling or version-scoping a rule' + ); + assert.match(stdout, /1 not compared: trigger \(no engine verdict\)/); +}); + +test('a leg where nothing could be compared is inconclusive, not agreement', () => { + const dir = writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }); + // Both cases lose their verdict: the whole leg proved nothing. + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', outcome: 'error', error: 'timeout' }, + ]) + ); + const { status, report, stdout } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + assert.equal(status, 1, 'inconclusive must be red — "could not check" is not "passed"'); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.drifts.length, 0, 'a dead leg must not manufacture linter advice'); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(stdout, /Inconclusive \(leg problem, not a linter problem\)/); +}); + +test('a reworded engine message does not mask a detector that went silent', () => { + // Regression: ENGINE_MESSAGE_CHANGED returned before the detector-silent check, + // so the report said "no rule change is required" while the rule had stopped + // firing. Re-pinning the string would have gone green over a dead rule. + const dir = writeLeg({ + version: '3.7.0', + cases: { + trigger: { detector: 0, rejected: true, reason: 'union needs >= 2 datasets' }, + control: { detector: 0, rejected: false }, + }, + }); + const { report } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + const drift = report.drifts.find((d) => d.queryName === 'trigger'); + assert.equal(drift.driftClass, 'detector-silent'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('a detector firing on a version its appliesTo excludes is reported', () => { + // OSD's version filter runs a rule when the cluster version is unknown, so an + // out-of-scope rule CAN reach users. Silence here would hide that false positive. + const dir = writeLeg({ + version: '3.6.0', // below the rule's 3.7 minVersion + cases: { trigger: { detector: 1, rejected: false }, control: { detector: 0, rejected: false } }, + }); + const { status, report } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + assert.equal(status, 1); + const drift = report.drifts.find((d) => d.version === '3.6.0'); + assert.equal(drift.driftClass, 'detector-noisy'); + assert.equal(drift.remediation.action, 'update-detector'); +}); + +test('a calcite-scoped expectation is selected rather than counted twice', () => { + // Both single-version halves drop `engine: "calcite"` entries when Calcite is + // off. Without that filter here, a per-engine pair for one range matches twice + // and is misreported as an uncovered version. + const contracts = writeContracts({ + expectations: [ + SPEC.expectations[0], + { ...SPEC.expectations[0], engine: undefined, queries: SPEC.expectations[0].queries }, + ], + }); + const { report } = run({ contracts, legs: healthyLegs() }); + // Two matching entries is genuinely ambiguous and must not silently pick one. + assert.ok( + report.coverageHoles.length > 0 || report.matrix.some((m) => m.status === 'uncovered'), + 'an ambiguous pair of expectations must be surfaced, not resolved arbitrarily' + ); +}); + test('a bad --leg argument is rejected', () => { const result = spawnSync( process.execPath, diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 63640958473..2ecb55d9964 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -228,6 +228,58 @@ test('a reworded rejection is update-contract and points at quoted copy', () => assert.match(drift.remediation.detail, /quotes the old engine wording/); }); +test('a reworded rejection does not mask a detector that stopped firing', () => { + // Regression: this branch used to return before the detector-silent check, so a + // simultaneous rewording + detector regression reported "the detector's verdict + // is unaffected, no rule change is required". Re-pinning the string would have + // turned the check green over a rule that no longer fires at all. + const drift = classifyDrift( + agreeingTrigger({ + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'union requires >= 2 datasets, got 1', + }, + expectedBackend: { + body: { + error: { + type: 'IllegalArgumentException', + reason: 'Union command requires at least two datasets. Provided: 1', + }, + }, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_SILENT); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); +}); + +test('a detector firing where appliesTo excludes the version is a false positive', () => { + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', // below the rule's 3.7 minVersion + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_NOISY); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + // The version filter's unknown-version behavior is why this reaches users. + assert.match(drift.remediation.detail, /version is unknown/); +}); + +test('an unobserved engine verdict is never treated as acceptance', () => { + // backendRejected: undefined means "we never got an answer". It must not select + // the engine-relaxed branch, which would advise disabling a healthy rule. + const drift = classifyDrift( + agreeingTrigger({ + observed: { detectorCount: 1, severities: ['error'], backendRejected: undefined }, + }) + ); + assert.equal(drift, null); +}); + test('a changed exception type is reported even when the reason is unchanged', () => { const drift = classifyDrift( agreeingTrigger({ diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 7b3f6d174fd..4ec33300975 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -194,6 +194,43 @@ function auditDefaultErrorCensus(legs, specs, enforcedRules) { return missing; } +/** + * Read one backend report entry into an observation, distinguishing "the engine + * accepted this" from "we never got an answer". + * + * This distinction is load-bearing. The IT marks a transport-level failure + * `outcome: "error"` and, having never received a verdict, writes no `rejected` + * field. Coercing that absence to `false` would report a timeout as an engine + * that now ACCEPTS a query it used to reject — which reads as an engine + * relaxation and would advise disabling a perfectly good rule. Anything that is + * not a real observed verdict becomes `undefined`, which the classifier treats as + * "not observed" rather than as acceptance. + * + * Returns `{ observed, usable }`: `usable` is false when this case produced no + * comparable engine verdict, so the caller can refuse to call it agreement. + */ +function readBackendObservation(backendEntry, detectorResult) { + const observedBackend = (backendEntry && backendEntry.observed) || undefined; + const outcome = backendEntry && backendEntry.outcome; + // `observed`/`error` are the observe-only outcomes; `pass`/`fail` come from the + // asserting mode. Only those carry a real verdict. + const hasVerdict = + !!backendEntry && + outcome !== 'error' && + (typeof backendEntry.rejected === 'boolean' || !!observedBackend); + + return { + usable: hasVerdict && !!detectorResult, + observed: { + detectorCount: detectorResult ? detectorResult.actual : 0, + severities: detectorResult ? detectorResult.severities || [] : [], + backendRejected: hasVerdict ? !!backendEntry.rejected : undefined, + backendType: observedBackend ? observedBackend.type : undefined, + backendReason: observedBackend ? observedBackend.reason : undefined, + }, + }; +} + /** * Check an out-of-scope rule for the one drift that still matters there: the * engine rejects a trigger query, but the rule's `appliesTo` excludes this @@ -244,9 +281,23 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { * "exactly one must match" rule as the two single-version halves. Returns * undefined when the corpus does not cover this version — reported separately as * a coverage hole, not as behavioral drift. + * + * The `engine` filter matters as much as the version range: both single-version + * halves (`PplLintRuleValidationIT.selectExpectation` and + * `run-frontend-contract.mjs`) drop `engine: "calcite"` entries when Calcite is + * off. Omitting it here would make a contract that pins one range per engine match + * TWICE and be misreported as an uncovered version. Every leg in this workflow + * runs with Calcite enabled (the observation legs do not disable it), so a + * calcite-scoped expectation is in play; a contract's `frontendContext.isCalcite: + * false` opts out. */ function selectExpectation(spec, version, versionMatchesRange) { - const matches = (spec.expectations || []).filter((exp) => versionMatchesRange(exp.version, version)); + const isCalcite = !((spec.frontendContext || {}).isCalcite === false); + const matches = (spec.expectations || []).filter((exp) => { + if (!versionMatchesRange(exp.version, version)) return false; + if (exp.engine === 'calcite' && !isCalcite) return false; + return true; + }); return matches.length === 1 ? matches[0] : undefined; } @@ -303,6 +354,10 @@ function main() { const drifts = []; const coverageHoles = []; + // Rule/version pairs where no case could actually be compared (a leg that lost + // its engine verdicts or its detector rows). Tracked separately from drift + // because the answer is "re-run / fix the leg", not "edit the linter". + const inconclusive = []; const matrix = []; // one row per rule × version, for the summary table // A rule that ships enabled at error severity but has no contract file is @@ -372,9 +427,17 @@ function main() { } let ruleDrifts = 0; + let compared = 0; + const unusable = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; - if (!queryDef) continue; // the single-version halves already fail on this + if (!queryDef) { + // The contract references a query it does not define. The single-version + // halves fail on this, but skipping it silently here would shrink the + // compared set without saying so. + unusable.push(`${queryName} (not defined in the contract's queries map)`); + continue; + } const query = queryDef.query.split('{{index}}').join(spec.index); const role = queryDef.role || 'trigger'; @@ -382,7 +445,19 @@ function main() { (r) => r.ruleId === ruleId && r.queryName === queryName ); const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); - const observedBackend = backendEntry && backendEntry.observed; + const { observed, usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + // No comparable pair, so there is nothing to classify. Attempting it + // anyway would turn a dead leg into linter advice: a case with no engine + // verdict and no detector row looks exactly like "the detector went + // silent", and the report would tell the engineer to go fix a detector + // that is fine. Record it as not compared and move on. + unusable.push( + `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` + ); + continue; + } + compared++; const drift = classifyDrift({ ruleId, @@ -395,13 +470,7 @@ function main() { severity: expected.severity, backendKind: expected.backend && expected.backend.kind, }, - observed: { - detectorCount: detectorResult ? detectorResult.actual : 0, - severities: detectorResult ? detectorResult.severities || [] : [], - backendRejected: backendEntry ? !!backendEntry.rejected : undefined, - backendType: observedBackend ? observedBackend.type : undefined, - backendReason: observedBackend ? observedBackend.reason : undefined, - }, + observed, wiring: spec.wiring, detectorPath: spec.detectorPath, parserRuleNames: leg.parserRuleNames, @@ -414,17 +483,39 @@ function main() { ruleDrifts++; } } - matrix.push({ - ruleId, - version: leg.version, - status: ruleDrifts === 0 ? 'agree' : 'drift', - drifts: ruleDrifts, - }); + // "agree" has to mean "we compared something and it matched". A rule whose + // every case lost its engine verdict (a timed-out leg) or its detector row + // (a runner that died mid-corpus) has proven nothing, and calling that + // agreement is exactly the vacuous pass this check exists to prevent. + if (compared === 0) { + inconclusive.push({ + ruleId, + file, + version: leg.version, + enforced: isEnforced, + reasons: unusable, + }); + matrix.push({ ruleId, version: leg.version, status: 'inconclusive', drifts: ruleDrifts }); + } else { + if (unusable.length > 0) { + log( + `WARN: ${ruleId} @ ${leg.version} compared ${compared} case(s); ` + + `${unusable.length} not compared: ${unusable.join(', ')}` + ); + } + matrix.push({ + ruleId, + version: leg.version, + status: ruleDrifts === 0 ? 'agree' : 'drift', + drifts: ruleDrifts, + }); + } } } const enforcedDrifts = drifts.filter((d) => d.enforced); const enforcedHoles = coverageHoles.filter((h) => h.enforced); + const enforcedInconclusive = inconclusive.filter((i) => i.enforced); const report = { schemaVersion: 1, @@ -439,15 +530,20 @@ function main() { matrix, drifts, coverageHoles, + inconclusive, result: { driftCount: drifts.length, enforcedDriftCount: enforcedDrifts.length, enforcedCoverageHoles: enforcedHoles.length, missingContractCount: missingContracts.length, + enforcedInconclusive: enforcedInconclusive.length, + // An inconclusive default-error rule fails too: "we could not check" must + // never render as "it is fine". passed: enforcedDrifts.length === 0 && enforcedHoles.length === 0 && - missingContracts.length === 0, + missingContracts.length === 0 && + enforcedInconclusive.length === 0, }, }; @@ -469,8 +565,8 @@ function main() { // eslint-disable-next-line no-console console.error( `[ppl-lint-multiversion] FAIL: ${enforcedDrifts.length} drift(s), ` + - `${enforcedHoles.length} coverage hole(s) and ${missingContracts.length} unvalidated ` + - `default-error rule(s).` + `${enforcedHoles.length} coverage hole(s), ${missingContracts.length} unvalidated ` + + `default-error rule(s) and ${enforcedInconclusive.length} inconclusive rule/version pair(s).` ); process.exit(1); } @@ -514,6 +610,21 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { } lines.push(''); + if ((report.inconclusive || []).length > 0) { + lines.push('### Inconclusive (leg problem, not a linter problem)'); + lines.push(''); + for (const entry of report.inconclusive) { + lines.push( + `- \`${entry.ruleId}\` on engine \`${entry.version}\`: no case could be compared — ` + + `${entry.reasons.join('; ')}. This is NOT a lint finding: the engine or the detector run ` + + `did not answer, so nothing was validated. Check that leg's job logs (an unreachable ` + + `cluster, an index that failed to seed, or a detector runner that died mid-corpus) and ` + + `re-run. Do not edit the rule or the contract on the strength of this.` + ); + } + lines.push(''); + } + if ((report.missingContracts || []).length > 0) { lines.push('### Unvalidated default-error rules'); lines.push(''); diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index b343778ac3e..62f4ad5b7d9 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -327,6 +327,34 @@ export function classifyDrift(input) { }, }; } + // Out of scope but the detector fired anyway. `appliesTo` is applied by OSD's + // version filter, which runs a rule when the cluster version is UNKNOWN — so a + // user whose version could not be resolved sees a diagnostic the catalog says + // does not apply to them. If the engine accepts the query, that is a false + // positive reaching exactly the users the version window was meant to protect. + if (detectorFlagged) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_NOISY, + evidence: + `${where}: the rule's appliesTo (${JSON.stringify((wiring && wiring.appliesTo) || {})}) ` + + `excludes ${version}, yet the detector emitted ${observed.detectorCount} diagnostic(s) ` + + `(${describeObservation(observed)}).`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `A rule out of scope for ${version} must stay silent there. Check that the detector honors ` + + `the version context rather than deciding on its own, and remember OSD's version filter runs ` + + `a rule when the cluster version is unknown — so this also fires for users whose version ` + + `could not be resolved.` + + (backendRejected === true + ? ` The engine does reject this query, so widening appliesTo in ${OSD_PATHS.catalog} may be` + + ` the right fix instead.` + : ''), + }, + }; + } // Out of scope and the engine agrees it is a non-issue: nothing to report. return null; } @@ -397,11 +425,19 @@ export function classifyDrift(input) { } // --- 4. Same verdict, different wording ------------------------------------ - // The engine still rejects, but the error type/reason moved. The detector is - // still right; the pinned body — and any detector text that quotes the engine - // wording — is stale. Worth flagging because linter messages and quick-fix - // copy are written against these strings. - if (backendRejected === true && expectRejection && expectedBackend) { + // The engine still rejects, but the error type/reason moved. The pinned body — + // and any detector text that quotes the engine wording — is stale. Worth + // flagging because linter messages and quick-fix copy are written against these + // strings. + // + // Requires the detector to still agree with the expectation (`detectorMatches`). + // Without that condition a reworded message would MASK a detector that went + // silent at the same time: the report would say "the detector's verdict is + // unaffected, no rule change required", the engineer would re-pin the string, + // and the check would go green over a rule that no longer fires. When both moved + // at once, the silent detector is the more serious story and step 5 tells it. + const detectorMatches = (observed.detectorCount || 0) === (expected.detectorCount || 0); + if (backendRejected === true && expectRejection && expectedBackend && detectorMatches) { const expectedError = (expectedBackend.body && expectedBackend.body.error) || {}; const typeChanged = expectedError.type !== undefined && From 2b505e9ccce888bcc4c1416f5b8eeeb3efe90192 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:01:41 -0700 Subject: [PATCH 04/39] fix(ci): forward ppl.lint.* system properties to the test JVM by prefix The multi-version matrix's first real run failed on the 3.6.0 leg with nine contract assertion failures. Root cause: `-Dppl.lint.observe.only=true` never reached the test JVM. integ-test/build.gradle forwards ppl.lint.* properties via a hand-maintained allowlist, and the new property was added to the IT and to the workflow but not to that list -- so the leg ASSERTED expectations pinned for 3.8 against a 3.6 engine, which is precisely the failure observe-only exists to prevent. Forward by prefix instead of by list. A hand-maintained list is a silent trap: an omitted property produces no error anywhere, it just quietly does nothing, and the resulting failure looks like a product bug rather than a plumbing bug. Any `ppl.lint.*` property the invoker sets now reaches the test JVM, so adding a knob to the IT is sufficient. Verified: `:integ-test:integTestRemote --dry-run` configures cleanly with -Dppl.lint.observe.only and -Dppl.lint.report set. Signed-off-by: Hanyu Wei --- integ-test/build.gradle | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 1d8f3af45f4..5dffd44560e 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -169,18 +169,25 @@ tasks.withType(licenseHeaders.class) { } // Forward the PPL lint rule validation contract knobs to every integ test JVM -// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), an -// optional path to write the observed backend report, and — while the cluster is -// alive — optional paths to export the candidate runtime grammar bundle and its -// target manifest for the detector-validation job. Applied globally so every +// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), +// whether to observe rather than assert (the multi-version matrix), an optional +// path to write the observed backend report, and — while the cluster is alive — +// optional paths to export the candidate runtime grammar bundle and its target +// manifest for the detector-validation job. Applied globally so every // RestIntegTestTask that runs the class picks it up without per-task edits. +// +// Forwarded by PREFIX rather than by an explicit list. A hand-maintained list is +// a silent trap: a property the IT reads but the list omits simply never reaches +// the test JVM, with no error anywhere. That already cost one CI run — +// `ppl.lint.observe.only` was added to the IT and the workflow but not to the +// list, so a multi-version leg asserted expectations pinned for a DIFFERENT +// engine version and failed instead of observing. Forwarding every `ppl.lint.*` +// property the invoker set means adding a knob to the IT is enough. tasks.withType(Test).configureEach { systemProperty "ppl.lint.schedule", System.getProperty("ppl.lint.schedule", "pr") - ["ppl.lint.report", "ppl.lint.grammar.bundle", "ppl.lint.target"].each { prop -> - if (System.getProperty(prop) != null) { - systemProperty prop, System.getProperty(prop) - } - } + System.properties.stringPropertyNames() + .findAll { it.startsWith("ppl.lint.") && it != "ppl.lint.schedule" } + .each { prop -> systemProperty prop, System.getProperty(prop) } } validateNebulaPom.enabled = false From 2edc319cb04f7837eb14452675c3ca0225036803 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:17:06 -0700 Subject: [PATCH 05/39] test(ppl-lint): pin per-version rejection shapes observed on 3.6 and 3.7 The first real multi-version run reported 5 drifts, all engine-message-changed. They were genuine version differences, not linter faults: the DETECTOR verdict is identical on 3.6/3.7/3.8 for every rule (each count matched, live-observed), but older engines describe the same rejection differently. Two epochs, both observed rather than guessed: - 3.6 masks the cause entirely. Every rejection collapses to HTTP 400 IllegalArgumentException / "Invalid Query" -- for capture-group names, unknown fields, flat_object references and replace wildcard mismatches alike. Pinning that string buys no discriminating power on its own, so on 3.6 these rules rely on the detector count for attribution; the note in each contract says so. - 3.7 already carries the specific wording 3.8 uses, so >=3.7.0 covers both. eventstats needs three epochs because its status also moved: 3.6 gives HTTP 500 UnsupportedOperationException / "There was internal problem at backend", 3.7 keeps the 500 but names the function, and 3.8 turned it into a proper HTTP 400 CalciteUnsupportedException. Keeping both epochs (rather than relaxing the newer expectation) is the point: the matrix now proves each rule still fires on older engines instead of reporting the wording difference as drift. Two reporting fixes found while reading the real output: an `inconclusive` matrix cell rendered BLANK because the status had no label (a blank cell reads as "nothing to see here", the opposite of what it means), and the headline counted only drifts and holes -- so a run failing solely on inconclusive rules displayed "0 drifts, 0 holes -- FAIL", which looks like a reporting bug rather than the actual cause. Unmapped statuses now render visibly and every failure reason is named in the headline. Verified by replaying all three real CI leg artifacts through the aggregator: 5 drifts -> 0, every version agreeing on all 5 rules whose detectors CI could load. 43 tests green. Signed-off-by: Hanyu Wei --- .../contracts/field-validation.spec.json | 84 +++++++++++++++++-- .../contracts/flat-object-subfield.spec.json | 81 ++++++++++++++++-- .../invalid-capture-group-name.spec.json | 59 +++++++++++-- .../replace-wildcard-asymmetry.spec.json | 60 +++++++++++-- ...ed-window-function-in-eventstats.spec.json | 51 ++++++++++- scripts/ppl-lint/aggregate-versions.mjs | 25 +++++- 6 files changed, 331 insertions(+), 29 deletions(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index c67343ef23a..45f85e3b769 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -13,12 +13,19 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true, - "visibleIndices": ["{{index}}"], + "visibleIndices": [ + "{{index}}" + ], "deriveFromMapping": { "account_number": "long", "balance": "long", @@ -50,7 +57,52 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "queries": { + "unknown-field-existence": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "grok-field-slot-shape-typo": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "known-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "queries": { "unknown-field-existence": { "detectorCount": 1, @@ -59,7 +111,13 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [nonexistent_field] not found." } } + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [nonexistent_field] not found." + } + } } }, "grok-field-slot-shape-typo": { @@ -68,12 +126,24 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Field [field] not found." } } + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [field] not found." + } + } } }, "known-field-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json index 25ac4cf66cf..440c55e27ac 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -3,7 +3,10 @@ "ruleId": "flat-object-subfield", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["qualifiedName", "wcQualifiedName"], + "requiredParserRules": [ + "qualifiedName", + "wcQualifiedName" + ], "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true).", "wiring": { "detector": "flat-object-subfield", @@ -15,8 +18,13 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["FLAT_OBJECT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "FLAT_OBJECT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true, @@ -47,7 +55,68 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "flat-object-dotted-subfield": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "flat-object-bare-root": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "flat-object-in-where": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "non-flat-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "engine": "calcite", "queries": { "flat-object-dotted-subfield": { @@ -100,7 +169,9 @@ "backend": { "kind": "result-shape", "httpStatus": 200, - "expect": { "datarowsNonEmpty": true } + "expect": { + "datarowsNonEmpty": true + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index c651af803b6..3d39fa2e3dc 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -3,7 +3,10 @@ "ruleId": "invalid-capture-group-name", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["rexCommand", "stringLiteral"], + "requiredParserRules": [ + "rexCommand", + "stringLiteral" + ], "wiring": { "detector": "invalid-capture-group-name", "enabled": true, @@ -14,12 +17,19 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true, - "deriveFromMapping": { "email": "text" } + "deriveFromMapping": { + "email": "text" + } }, "index": "opensearch-sql_test_index_account", "queries": { @@ -34,7 +44,38 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "rex-capture-name-underscore": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "rex-capture-name-alphanumeric-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "engine": "calcite", "queries": { "rex-capture-name-underscore": { @@ -54,7 +95,13 @@ }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index 8946dad60a9..593bb92023d 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -3,7 +3,10 @@ "ruleId": "replace-wildcard-asymmetry", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["replacePair", "stringLiteral"], + "requiredParserRules": [ + "replacePair", + "stringLiteral" + ], "wiring": { "detector": "replace-wildcard-asymmetry", "enabled": true, @@ -11,11 +14,19 @@ "runtimeOnly": true, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.4.0", "engine": "calcite" } + "appliesTo": { + "minVersion": "3.4.0", + "engine": "calcite" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -33,7 +44,38 @@ }, "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": { + "replace-wildcard-count-mismatch": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, + "replace-symmetric-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, + { + "version": ">=3.7.0", "engine": "calcite", "queries": { "replace-wildcard-count-mismatch": { @@ -53,7 +95,13 @@ }, "replace-symmetric-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 6f6fbd313b0..d6821273175 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -31,9 +31,58 @@ "query": "source={{index}} | eventstats avg(age) as avg_age" } }, + "notes": "The DETECTOR verdict is identical on 3.6/3.7/3.8 (1 diagnostic on the trigger, 0 on the control, live-observed). Only the engine's rejection shape moves, in three epochs: 3.6 masks the cause entirely (HTTP 500 UnsupportedOperationException / 'There was internal problem at backend'); 3.7 keeps the 500 but names the function; 3.8 turned it into a proper HTTP 400 CalciteUnsupportedException. Each epoch is pinned separately so the multi-version check proves the rule still fires on older engines instead of reporting the wording difference as drift. A 500 is a poor rejection oracle (it cannot distinguish this failure from an unrelated crash), which is why the <3.8 entries additionally rely on the detector count for discrimination.", "expectations": [ { - "version": ">=3.4.0", + "version": ">=3.4.0 <3.7.0", + "queries": { + "eventstats-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + }, + "eventstats-avg-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + }, + { + "version": ">=3.7.0 <3.8.0", + "queries": { + "eventstats-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "Unexpected window function: rank" + } + } + } + }, + "eventstats-avg-control": { + "detectorCount": 0, + "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } + } + }, + { + "version": ">=3.8.0", "queries": { "eventstats-rank": { "detectorCount": 1, diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 4ec33300975..9efc55bd850 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -582,11 +582,23 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { const lines = []; lines.push('## PPL lint multi-version validation'); lines.push(''); + // Every reason the run can be red belongs in the headline. Reporting only + // drifts and holes made a FAIL caused solely by inconclusive rules read as + // "0 drifts, 0 holes — FAIL", which looks like a reporting bug rather than the + // real cause. + const reasons = [ + `${report.result.enforcedDriftCount} enforced drift(s)`, + `${report.result.enforcedCoverageHoles} coverage hole(s)`, + ]; + if (report.result.enforcedInconclusive) { + reasons.push(`${report.result.enforcedInconclusive} inconclusive`); + } + if (report.result.missingContractCount) { + reasons.push(`${report.result.missingContractCount} unvalidated rule(s)`); + } lines.push( `Engine versions: ${legs.map((l) => `\`${l.version}\``).join(', ')} — ` + - `**${report.result.passed ? 'PASS' : 'FAIL'}** ` + - `(${report.result.enforcedDriftCount} enforced drift(s), ` + - `${report.result.enforcedCoverageHoles} coverage hole(s))` + `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` ); lines.push(''); @@ -599,12 +611,17 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { drift: 'DRIFT', uncovered: 'not covered', 'out-of-scope': 'n/a (out of scope)', + inconclusive: '**inconclusive**', }; for (const ruleId of rules) { const cells = versions.map((version) => { const row = report.matrix.find((m) => m.ruleId === ruleId && m.version === version); if (!row) return '—'; - return row.status === 'drift' ? `**DRIFT** (${row.drifts})` : cell[row.status]; + if (row.status === 'drift') return `**DRIFT** (${row.drifts})`; + // An unmapped status must still render as something visible. A blank cell + // reads as "nothing to see here", which is the opposite of what an + // unrecognized state means. + return cell[row.status] || `**${row.status}**`; }); lines.push(`| \`${ruleId}\` | ${cells.join(' | ')} |`); } From 33a3ffed5521333d8343d29f7ef77d85d760d5e3 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:50:18 -0700 Subject: [PATCH 06/39] fix(ci): treat a missing version expectation as an observation, not a failure With the property plumbing fixed, the 3.6 leg went from nine contract failures to one: "[union-min-datasets] no version expectation matches backend version 3.6.0". union-min-datasets is a >=3.7 rule, so having no 3.6 expectation is correct -- the aggregator already classifies that as out-of-scope. But observe-only mode could not suppress it, because selectExpectation records into whatever failure list it is handed and was handed the real one. The leg therefore recorded the observation AND failed on it, which is precisely the "a disagreement is the signal, not a broken run" contract that observe-only exists to honor. Hand selectExpectation a scratch list in observe-only mode and discard it. The two other failure sites are deliberately left failing even in observe-only: a contract referencing an undefined query, and a grammar bundle that could not be exported, are both broken-run problems rather than engine behavior, and silencing them would produce an empty report that reads as agreement. The same run confirmed the earlier work landed: invalid-capture-group-name now PASSes on 3.6 against its new generic-rejection epoch, and the 3.7.0 and pr-build legs both went green. Signed-off-by: Hanyu Wei --- .../calcite/remote/PplLintRuleValidationIT.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 065bc4e78eb..7ae505e9b35 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -180,14 +180,20 @@ private void runContract( List applied = applyClusterSettings(fixture); try { - JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, failures); + // In observe-only mode, "no expectation matches this version" is information, + // not a failure — a rule the corpus does not pin for THIS engine is exactly + // what the multi-version matrix is here to learn. selectExpectation records + // into whatever list it is handed, so hand it a scratch list we discard; + // otherwise the leg both records the observation AND fails, which is what + // kept union-min-datasets (a >=3.7 rule) failing the 3.6 leg. + List selectionFailures = observeOnly ? new ArrayList<>() : failures; + JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, selectionFailures); if (selected == null) { if (!observeOnly) { return; // no/ambiguous version expectation — failure already recorded. } - // Observe-only: an engine the corpus does not pin is exactly what the - // multi-version matrix wants to learn about, so record the raw behavior of - // every query and let the aggregator decide whether the gap matters. + // Record the raw behavior of every query and let the aggregator decide + // whether the gap matters (out-of-scope rule vs a real coverage hole). observeAllQueries(ruleId, index, queries, report); return; } From c2920cc20e74a7bc7199e0987e13d8a0c8c5143e Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sat, 25 Jul 2026 20:55:36 -0700 Subject: [PATCH 07/39] fix(ci): do not report an unsupported command as a too-narrow version window With all 7 default-error rules finally loading, the matrix came back 20/21 cells agreeing, with one drift: version-scope-too-narrow for union-min-datasets on 3.6, advising "widen appliesTo to include 3.6.0". That advice was wrong. On 3.6 the `union` command does not exist at all -- the rule's CONTROL query (a valid two-dataset union) is rejected with the same SyntaxCheckException as the single-dataset trigger. The classifier judged the trigger in isolation, so a rejection it read as "the engine has this behavior, your rule just is not scoped to it" was really "this command is unsupported here". Following it would ship a diagnostic claiming a precise cause ("union requires at least two datasets") for a query that fails simply because the command is unknown. Pass whether the control was also rejected. When it was, the version window is doing its job and the pair is correctly reported out-of-scope rather than as drift. This is the same failure mode as the earlier vacuous-pass fixes, in the opposite direction: not a missed finding, but a confidently WRONG remediation. A rejection only means what the rule claims if a comparable valid query succeeds. Verified by replaying all three real CI leg artifacts: the matrix is now fully green -- 7 default-error rules x 3 engine versions (3.6.0, 3.7.0, 3.8.0-SNAPSHOT), 0 drifts, 0 coverage holes, 0 inconclusive. 44 tests green. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/__tests__/drift.test.mjs | 22 ++++++++++++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 12 ++++++++++++ scripts/ppl-lint/drift.mjs | 12 +++++++++++- 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 2ecb55d9964..4c8d4e6f4dc 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -368,6 +368,28 @@ test('a downgraded severity is caught even when the count is right', () => { // --- version scoping -------------------------------------------------------- +test('an unsupported command is not mistaken for a too-narrow version window', () => { + // Real case from CI: on 3.6 the `union` command does not exist, so BOTH the + // trigger and the control fail with SyntaxCheckException. Judging the trigger + // alone said "widen appliesTo to 3.6" — which would ship a diagnostic claiming a + // precise cause ("requires at least two datasets") for what is really + // "unsupported command". A rejected control means the version window is right. + const drift = classifyDrift( + agreeingTrigger({ + version: '3.6.0', + observed: { + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'Invalid Query', + }, + controlAlsoRejected: true, + }) + ); + assert.equal(drift, null); +}); + test('an out-of-scope rule on an engine that rejects is scoped too narrowly', () => { const drift = classifyDrift( agreeingTrigger({ diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 9efc55bd850..5ff5c513a22 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -243,6 +243,17 @@ function readBackendObservation(backendEntry, detectorResult) { */ function classifyOutOfScope({ spec, ruleId, leg, classify }) { const found = []; + + // Did this rule's CONTROL query — a valid use of the same command — also get + // rejected on this engine? If so the command itself is unsupported here, and a + // rejected trigger says nothing about the rule's specific condition. Computed + // once, since it is a property of the rule on this engine. + const controlAlsoRejected = Object.entries(spec.queries || {}).some(([name, def]) => { + if ((def.role || 'trigger') !== 'control') return false; + const entry = leg.backend.get(`${ruleId}::${name}`); + return !!(entry && entry.rejected); + }); + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { if ((queryDef.role || 'trigger') !== 'trigger') continue; const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); @@ -268,6 +279,7 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { }, wiring: spec.wiring, detectorPath: spec.detectorPath, + controlAlsoRejected, // Deliberately no parser-rule check here: a grammar that lacks the rule is // expected on an engine the command predates. }); diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index 62f4ad5b7d9..43c9979bb6d 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -261,6 +261,8 @@ export function classifyGrammarDrift({ * @param {string[]} [input.parserRuleNames] candidate grammar's parser rule names * @param {string[]} [input.requiredParserRules] grammar rules the detector walks * @param {object} [input.expectedBackend] contract's pinned rejection body + * @param {boolean} [input.controlAlsoRejected] true when this rule's control query was + * ALSO rejected on this engine, i.e. the command itself is unsupported here */ export function classifyDrift(input) { const { @@ -276,6 +278,7 @@ export function classifyDrift(input) { requiredParserRules, expectedBackend, detectorPath, + controlAlsoRejected, } = input; const detectorFlagged = (observed.detectorCount || 0) > 0; @@ -310,7 +313,14 @@ export function classifyDrift(input) { // this version get no diagnostic. const inScope = versionInAppliesTo(wiring && wiring.appliesTo, version); if (!inScope) { - if (role === 'trigger' && backendRejected === true) { + // A trigger the engine rejects normally means the version window is too + // narrow. But if the rule's CONTROL — a valid query using the same command — + // is rejected too, the command itself does not exist on this engine yet, and + // the rejection says nothing about the rule's specific condition. Widening + // appliesTo there would ship a diagnostic that claims a precise cause for + // what is really "unsupported command", so that case is correctly silent: + // the version window is doing its job. + if (role === 'trigger' && backendRejected === true && controlAlsoRejected !== true) { return { ...base, driftClass: DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW, From 03ae9ed7912031d897fba0220905c6d4406334ce Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 12:38:40 -0700 Subject: [PATCH 08/39] fix(ci): apply the unobserved-verdict rule to the out-of-scope path too Code review of this branch found three defects, all with one root cause: the "an absent verdict is not a verdict" protection added earlier lives in readBackendObservation, which only the IN-SCOPE path used. The out-of-scope path re-read `entry.rejected` inline and so lost it. 1. An errored trigger on an out-of-scope rule coerced to `rejected: false`, and the version-scope-too-narrow check needs `=== true` -- so a genuinely mis-scoped rule (3.6 rejects the trigger, 3.6 users get no diagnostic) rendered as `out-of-scope` with exit 0. Reachable via the new recordUnusableContract path: an unseeded fixture marks every case errored, so a version-scoped rule validates nothing and still shows green. 2. `controlAlsoRejected` failed OPEN for the same reason, producing exactly the wrong advice it was written to prevent -- "lower union-min-datasets' minVersion to 3.6", shipping "union requires two datasets" to users whose query fails only because the command is absent. Fixed by making the control verdict THREE-state: rejected (command unsupported -> suppress), accepted (command works -> report), unknown (no verdict -> cannot tell, so stay quiet). Collapsing it to a boolean was the bug. 3. `compared === 0` let a partially-observed rule render `agree`. flat-object-subfield has three triggers and one control; if all three triggers lost their verdict and only the control survived, `compared === 1` and the cell said `agree`. Triggers ARE the rule's behavioral claim, so they are now counted separately and losing all of them is inconclusive. Verified: the genuine version-scope-too-narrow finding still fires (only unobserved cases are suppressed, not real ones), and replaying all three real CI leg artifacts is still fully green -- 7 default-error rules x 3 engine versions. 47 tests, up from 44; one existing test's tail assertion was updated because losing a single-trigger spec's only trigger now correctly reports inconclusive rather than passing with a warning. Signed-off-by: Hanyu Wei --- .../__tests__/aggregate-versions.test.mjs | 97 ++++++++++++++++++- scripts/ppl-lint/aggregate-versions.mjs | 77 ++++++++++----- 2 files changed, 149 insertions(+), 25 deletions(-) diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 76e7b558181..65606dbb926 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -368,7 +368,7 @@ test('a transport error is not read as engine acceptance', () => { cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, }), }; - const { report, stdout } = run({ contracts: writeContracts(), legs }); + const { status, report, stdout } = run({ contracts: writeContracts(), legs }); assert.equal( report.drifts.filter((d) => d.driftClass === 'engine-relaxed').length, 0, @@ -378,7 +378,39 @@ test('a transport error is not read as engine acceptance', () => { !/FALSE POSITIVE/.test(stdout), 'a timeout must never advise disabling or version-scoping a rule' ); - assert.match(stdout, /1 not compared: trigger \(no engine verdict\)/); + // This spec has a single trigger, so losing it means the rule's behavioral + // claim went unchecked: inconclusive and red, not a passing WARN. + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.match(stdout, /trigger \(no engine verdict\)/); +}); + +test('losing every trigger is inconclusive even when a control still compares', () => { + // flat-object-subfield's real shape: several triggers plus one control. The + // triggers ARE the rule's claim, so a leg that kept only the control has proven + // nothing — but `compared > 0`, so a naive count would have rendered `agree`. + const dir = writeLeg({ + version: '3.7.0', + cases: { trigger: { detector: 1, rejected: true }, control: { detector: 0, rejected: false } }, + }); + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + rejected: false, + outcome: 'observed', + observed: { httpStatus: 200, rejected: false }, + }, + ]) + ); + const { status, report } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); + assert.equal(status, 1, 'a rule whose triggers all went unobserved must not read as agreement'); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.equal(report.result.enforcedInconclusive, 1); }); test('a leg where nothing could be compared is inconclusive, not agreement', () => { @@ -451,6 +483,67 @@ test('a calcite-scoped expectation is selected rather than counted twice', () => ); }); +test('an errored trigger on an out-of-scope rule does not silently pass', () => { + // The out-of-scope path used to read `entry.rejected` directly. An errored + // observation has no such field, so it coerced to false, the + // version-scope-too-narrow check (which needs `=== true`) never fired, and a + // genuinely mis-scoped rule rendered as `out-of-scope` with exit 0. + const dir = writeLeg({ + version: '3.6.0', // below the rule's 3.7 minVersion => out of scope + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: false } }, + }); + fs.writeFileSync( + path.join(dir, 'backend-report.json'), + JSON.stringify([ + { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + rejected: false, + outcome: 'observed', + observed: { httpStatus: 200, rejected: false }, + }, + ]) + ); + const { report } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + // The point is that an unobserved trigger yields no CLAIM either way: it must + // not be reported as a confident out-of-scope agreement... + assert.equal( + report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow').length, + 0, + 'an unobserved trigger cannot support a version-scope finding' + ); + // ...nor may it invent linter advice from a verdict that never arrived. + assert.equal(report.drifts.length, 0); +}); + +test('an errored control cannot fail open into "widen appliesTo" advice', () => { + // controlAlsoRejected suppresses the version-scope finding when the command + // itself is unsupported. Reading `entry.rejected` raw made that suppression fail + // OPEN on an errored control: the run would then advise lowering minVersion, + // shipping a precise-cause diagnostic for an unknown-command failure. + const dir = writeLeg({ + version: '3.6.0', + cases: { trigger: { detector: 0, rejected: true }, control: { detector: 0, rejected: true } }, + }); + const backend = JSON.parse(fs.readFileSync(path.join(dir, 'backend-report.json'), 'utf8')).map( + (e) => + e.role === 'control' + ? { ruleId: e.ruleId, queryName: e.queryName, role: e.role, outcome: 'error', error: 'timeout' } + : { ...e, outcome: 'observed' } + ); + fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); + const { report, stdout } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + const scoped = report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow'); + assert.equal( + scoped.length, + 0, + 'with the control unobserved there is no evidence the command is supported, so no widening advice' + ); + assert.ok(!/Widen "/.test(stdout)); +}); + test('a bad --leg argument is rejected', () => { const result = spawnSync( process.execPath, diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 5ff5c513a22..2f49bf5b3b8 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -244,24 +244,42 @@ function readBackendObservation(backendEntry, detectorResult) { function classifyOutOfScope({ spec, ruleId, leg, classify }) { const found = []; - // Did this rule's CONTROL query — a valid use of the same command — also get - // rejected on this engine? If so the command itself is unsupported here, and a - // rejected trigger says nothing about the rule's specific condition. Computed - // once, since it is a property of the rule on this engine. - const controlAlsoRejected = Object.entries(spec.queries || {}).some(([name, def]) => { - if ((def.role || 'trigger') !== 'control') return false; - const entry = leg.backend.get(`${ruleId}::${name}`); - return !!(entry && entry.rejected); - }); + // What did this rule's CONTROL queries — valid uses of the same command — do on + // this engine? THREE states, not two, and the difference decides whether a + // rejected trigger means anything: + // rejected the command itself is unsupported here, so the trigger's rejection + // says nothing about the rule's specific condition -> suppress + // accepted the command works, so a rejected trigger really is the rule's + // condition going unreported on this version -> report it + // unknown no control verdict arrived (errored/absent). We cannot tell the two + // apart, so we must not emit confident advice either way. + // Collapsing this to a boolean is what let the suppression fail open: an errored + // control read as "not rejected" and produced the exact "widen appliesTo" advice + // this check exists to prevent. + const controlVerdicts = Object.entries(spec.queries || {}) + .filter(([, def]) => (def.role || 'trigger') === 'control') + .map(([name]) => { + const entry = leg.backend.get(`${ruleId}::${name}`); + const { observed } = readBackendObservation(entry, { actual: 0, severities: [] }); + return observed.backendRejected; + }); + const controlAlsoRejected = controlVerdicts.some((v) => v === true); + // A rule with controls, none of which produced a verdict, cannot be judged here. + const controlUnknown = + controlVerdicts.length > 0 && !controlVerdicts.some((v) => typeof v === 'boolean'); for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { if ((queryDef.role || 'trigger') !== 'trigger') continue; const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); if (!backendEntry) continue; // this leg never ran the query - const observedBackend = backendEntry.observed || {}; const detectorResult = (leg.detector.results || []).find( (r) => r.ruleId === ruleId && r.queryName === queryName ); + // Same reason as above: an errored observation must not read as "the engine + // accepted this". On this path that coercion would turn a genuinely + // mis-scoped rule into a silent `out-of-scope` PASS, because the + // version-scope-too-narrow check requires backendRejected === true. + const { observed: outOfScopeObserved } = readBackendObservation(backendEntry, detectorResult); const drift = classify({ ruleId, version: leg.version, @@ -270,16 +288,13 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { query: queryDef.query.split('{{index}}').join(spec.index), // Out of scope means the rule is expected to stay silent here. expected: { detectorCount: 0 }, - observed: { - detectorCount: detectorResult ? detectorResult.actual : 0, - severities: detectorResult ? detectorResult.severities || [] : [], - backendRejected: !!backendEntry.rejected, - backendType: observedBackend.type, - backendReason: observedBackend.reason, - }, + observed: outOfScopeObserved, wiring: spec.wiring, detectorPath: spec.detectorPath, - controlAlsoRejected, + // An unknown control verdict is treated the same as a rejected one: both + // mean "we cannot claim this engine supports the command", and staying quiet + // is the only honest option. + controlAlsoRejected: controlAlsoRejected || controlUnknown, // Deliberately no parser-rule check here: a grammar that lacks the rule is // expected on an engine the command predates. }); @@ -440,6 +455,14 @@ function main() { let ruleDrifts = 0; let compared = 0; + // Triggers are counted separately from controls. A trigger is the rule's + // entire behavioral claim ("this query is flagged"); a control only says the + // rule stays quiet nearby. So a rule that lost every trigger but kept one + // control has proven nothing about itself, even though `compared` is + // non-zero — flat-object-subfield has 3 triggers and 1 control, and would + // otherwise render `agree` off the control alone. + let triggersExpected = 0; + let triggersCompared = 0; const unusable = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; @@ -452,6 +475,9 @@ function main() { } const query = queryDef.query.split('{{index}}').join(spec.index); const role = queryDef.role || 'trigger'; + if (role === 'trigger') { + triggersExpected++; + } const detectorResult = (leg.detector.results || []).find( (r) => r.ruleId === ruleId && r.queryName === queryName @@ -470,6 +496,9 @@ function main() { continue; } compared++; + if (role === 'trigger') { + triggersCompared++; + } const drift = classifyDrift({ ruleId, @@ -495,11 +524,13 @@ function main() { ruleDrifts++; } } - // "agree" has to mean "we compared something and it matched". A rule whose - // every case lost its engine verdict (a timed-out leg) or its detector row - // (a runner that died mid-corpus) has proven nothing, and calling that - // agreement is exactly the vacuous pass this check exists to prevent. - if (compared === 0) { + // "agree" has to mean "we compared the rule's claim and it held". A rule + // whose every case lost its engine verdict (a timed-out leg) or its detector + // row (a runner that died mid-corpus) has proven nothing — and so has one + // that lost every TRIGGER while keeping a control, since the triggers are + // where the rule's behavior actually lives. Calling either agreement is the + // vacuous pass this check exists to prevent. + if (compared === 0 || (triggersExpected > 0 && triggersCompared === 0)) { inconclusive.push({ ruleId, file, From 052fbb190ae7520daa420bbee93050b47df48a8e Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 19:21:59 -0700 Subject: [PATCH 09/39] =?UTF-8?q?test(ci):=20TEMPORARY=20drift=20probe=20?= =?UTF-8?q?=E2=80=94=20do=20not=20merge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deliberately stale the >=3.7.0 pinned rejection reason for invalid-capture-group-name so the multi-version check has a real behavior change to classify. Both the 3.7.0 and pr-build legs reject this trigger with "Invalid capture group name 'user_name'."; this pins the truncated wording. Expected: engine-message-changed / update-contract on two engine legs, and a red Detect job. Reverted immediately after the run. Signed-off-by: Hanyu Wei --- .../ppl-lint/contracts/invalid-capture-group-name.spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index 3d39fa2e3dc..d2ca4dca8d2 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -88,7 +88,7 @@ "status": 400, "error": { "type": "IllegalArgumentException", - "reason": "Invalid capture group name 'user_name'." + "reason": "Invalid capture group name" } } } From 7cf8df2f1e62bbff76fb8b4b5229793d3a25d227 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 19:34:52 -0700 Subject: [PATCH 10/39] =?UTF-8?q?Revert=20"test(ci):=20TEMPORARY=20drift?= =?UTF-8?q?=20probe=20=E2=80=94=20do=20not=20merge"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 052fbb190ae7520daa420bbee93050b47df48a8e. Signed-off-by: Hanyu Wei --- .../ppl-lint/contracts/invalid-capture-group-name.spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index d2ca4dca8d2..3d39fa2e3dc 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -88,7 +88,7 @@ "status": 400, "error": { "type": "IllegalArgumentException", - "reason": "Invalid capture group name" + "reason": "Invalid capture group name 'user_name'." } } } From 35ed509795fba861f5595e32034ed5cfb04fdf48 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 19:55:32 -0700 Subject: [PATCH 11/39] feat(ci): surface PPL lint drift as GitHub annotations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The drift report already says what to change, but it only reached the job summary. GitHub renders annotations at the top of the run page, and the only one a failing multi-version run produced was "Process completed with exit code 1" — so the natural next click went to raw job logs instead of the remediation. Emit each finding as a workflow command before the summary. An update-contract finding anchors on the exact expectations[] entry whose version range produced it, so when the contract is part of the PR's diff the drift also attaches inline to the line that caused it. Rule-wide findings (a renamed grammar rule) anchor on the contract's ruleId instead. A line number is emitted only when it is unambiguous: a contract that pins the same range twice, or a range that cannot be found, yields file-only. A wrong line sends the reader to edit the wrong expectation, which is worse than making them find it. Severity carries meaning. Inconclusive findings are warnings, not errors: they mean the leg did not answer, the run is already red from the exit code, and listing them beside real drift invites editing a rule because a leg timed out. Non-enforced drift is a warning for the same reason it does not fail the run. Verified against the real leg artifacts from a multi-version run (three engine versions, three distinct grammar hashes): a clean corpus emits nothing, and a stale pinned reason annotates the correct expectation line on both affected engine legs. 12 new tests; 59 green. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/README.md | 31 +++ scripts/ppl-lint/__tests__/annotate.test.mjs | 193 ++++++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 24 +- scripts/ppl-lint/annotate.mjs | 227 +++++++++++++++++++ 4 files changed, 474 insertions(+), 1 deletion(-) create mode 100644 scripts/ppl-lint/__tests__/annotate.test.mjs create mode 100644 scripts/ppl-lint/annotate.mjs diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 76b3d56a0e0..0faebbb57ff 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -266,6 +266,37 @@ engine also accepts is reported as `n/a (out of scope)`, not as drift — that i the version window working. But if the engine *rejects* the trigger there, it is `version-scope-too-narrow`. +### Where a failure shows up in the GitHub UI + +Every finding is emitted twice, because the run page and the diff are two +different places a developer looks: + +1. **Annotations** (top of the run page, and inline on the file in *Files + changed* when the contract is part of the PR's diff). Each carries the drift + class, the rule, the engine version, and the one-line action. An + `update-contract` finding anchors on the exact `expectations[]` entry whose + `version` range produced it — not the top of the file — so the drift appears on + the line that caused it. Rule-wide findings (a renamed grammar rule) anchor on + the contract's `ruleId` instead. +2. **The job summary** — the rule × version table plus the full grouped + remediation report, which stays the authoritative account. + +Without the annotations the only thing above the summary is `Process completed +with exit code 1`, so the natural next click lands in raw job logs rather than the +remediation. Severity is not cosmetic: + +| Finding | Level | Why | +| --- | --- | --- | +| enforced drift, coverage hole | `error` | a shipped default-error rule disagrees with a supported engine | +| non-enforced drift | `warning` | reported, but it does not block | +| `inconclusive` | `warning` | "we could not check" is a leg problem; the run is already red from the exit code, and rendering it as an error invites editing a rule because a leg timed out | +| unvalidated default-error rule | `error` (no file) | the edit goes in `manifest.json`, not a contract | + +A line number is emitted only when it is unambiguous. If a contract pins the same +version range twice, or the range cannot be found, the annotation carries the file +and no line — a wrong line sends the reader to edit the wrong expectation, which +is worse than making them find it. + ### Running the multi-version check locally Each leg needs a reachable cluster. Point the observe step at any running engine: diff --git a/scripts/ppl-lint/__tests__/annotate.test.mjs b/scripts/ppl-lint/__tests__/annotate.test.mjs new file mode 100644 index 00000000000..dc55503a6ad --- /dev/null +++ b/scripts/ppl-lint/__tests__/annotate.test.mjs @@ -0,0 +1,193 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + buildAnnotations, + contractRepoPath, + findExpectationLine, + findRuleIdLine, + formatAnnotation, +} from '../annotate.mjs'; + +/** A contract shaped like the real ones, with two version-scoped expectations. */ +const CONTRACT = `{ + "schemaVersion": 3, + "ruleId": "invalid-capture-group-name", + "queries": { + "trigger": { "role": "trigger", "query": "source={{index}} | rex ..." } + }, + "expectations": [ + { + "version": ">=3.4.0 <3.7.0", + "engine": "calcite", + "queries": {} + }, + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": {} + } + ] +} +`; + +const readStub = (text) => () => text; + +test('anchors on the expectation entry that drifted, not the first one', () => { + assert.equal(findExpectationLine(CONTRACT, '>=3.7.0'), 14); + assert.equal(findExpectationLine(CONTRACT, '>=3.4.0 <3.7.0'), 9); +}); + +test('an ambiguous or absent range yields no line rather than a wrong one', () => { + // A wrong line number sends the reader to edit the wrong expectation, which is + // worse than making them find it: prefer no anchor. + const duplicated = CONTRACT.replace('">=3.4.0 <3.7.0"', '">=3.7.0"'); + assert.equal(findExpectationLine(duplicated, '>=3.7.0'), undefined); + assert.equal(findExpectationLine(CONTRACT, '>=9.9.9'), undefined); + assert.equal(findExpectationLine(undefined, '>=3.7.0'), undefined); + assert.equal(findExpectationLine(CONTRACT, undefined), undefined); +}); + +test('incidental whitespace does not defeat the anchor', () => { + const spaced = CONTRACT.replace('"version": ">=3.7.0"', '"version": ">=3.7.0"'); + assert.equal(typeof findExpectationLine(spaced, '>=3.7.0'), 'number'); +}); + +test('falls back to the ruleId line for rule-wide findings', () => { + assert.equal(findRuleIdLine(CONTRACT), 3); +}); + +test('a drift with no expectation range still anchors at the rule', () => { + // grammar-rule-missing is a fact about the rule on that engine, so it carries no + // expectationRange — it must still land on the file at a usable line. + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + driftClass: 'grammar-rule-missing', + enforced: true, + contractFile: 'invalid-capture-group-name.spec.json', + evidence: 'the candidate grammar has no parser rule(s) "rexCommand"', + remediation: { action: 'update-detector', detail: 'Re-anchor the detector.' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].line, 3); + assert.equal(annotations[0].level, 'error'); +}); + +test('inconclusive findings are warnings, never errors', () => { + // "We could not check" must not sit in the error list beside real drift, or the + // reader edits a rule because a leg timed out. + const annotations = buildAnnotations( + { + inconclusive: [ + { + ruleId: 'field-validation', + version: '3.6.0', + enforced: true, + file: 'field-validation.spec.json', + reasons: ['unknown-field-existence (no engine verdict)'], + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].level, 'warning'); + assert.match(annotations[0].message, /NOT a lint finding/); + assert.match(annotations[0].message, /Do not edit the rule/); +}); + +test('a non-enforced drift is a warning so it cannot be read as blocking', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'head-without-sort', + version: '3.8.0', + driftClass: 'detector-noisy', + enforced: false, + contractFile: 'head-without-sort.spec.json', + evidence: 'evidence', + remediation: { action: 'update-detector', detail: 'detail' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations[0].level, 'warning'); +}); + +test('an unvalidated rule has no file to point at', () => { + const annotations = buildAnnotations( + { missingContracts: [{ ruleId: 'sort-on-eval-field', reason: 'has no contract file' }] }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].file, undefined); + assert.equal(annotations[0].level, 'error'); + assert.match(annotations[0].message, /manifest\.defaultError/); +}); + +test('paths are repo-relative so GitHub can render them inline', () => { + // An absolute path still annotates the run, but never attaches to the diff. + assert.equal( + contractRepoPath('/w/integ-test/res/contracts', 'a.spec.json', '/w'), + 'integ-test/res/contracts/a.spec.json' + ); + // No workspace (a local run): absolute is the honest answer. + assert.equal(contractRepoPath('/w/c', 'a.spec.json', undefined), '/w/c/a.spec.json'); +}); + +test('workflow-command metacharacters are escaped', () => { + const line = formatAnnotation({ + level: 'error', + file: 'a,b:c.json', + line: 12, + title: 'has: comma, and colon', + message: 'first\nsecond 100% done', + }); + // Commas/colons in properties would otherwise terminate the property list. + assert.match(line, /file=a%2Cb%3Ac\.json/); + assert.match(line, /title=has%3A comma%2C and colon/); + // Newlines must survive as %0A or the annotation is truncated to one line. + assert.match(line, /first%0Asecond 100%25 done/); + assert.ok(line.startsWith('::error ')); +}); + +test('an unreadable contract still produces a file-less annotation', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'r', + version: '3.8.0', + driftClass: 'detector-silent', + enforced: true, + contractFile: 'gone.spec.json', + evidence: 'evidence', + remediation: { action: 'update-detector', detail: 'detail' }, + }, + ], + }, + { contractsDir: '/w/c', workspace: '/w', readFile: () => undefined } + ); + assert.equal(annotations.length, 1); + assert.equal(annotations[0].line, undefined); + assert.equal(annotations[0].file, 'c/gone.spec.json'); +}); + +test('a clean report emits nothing', () => { + assert.deepEqual(buildAnnotations({}, { contractsDir: '/w/c', workspace: '/w' }), []); +}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 2f49bf5b3b8..83bb5bd39e1 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -33,6 +33,7 @@ import fs from 'fs'; import path from 'path'; +import { emitAnnotations } from './annotate.mjs'; import { classifyDrift, classifyGrammarDrift, @@ -520,7 +521,17 @@ function main() { }); if (drift) { - drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + // `expectationRange` is what the annotation anchors to: the version + // string identifies WHICH `expectations[]` entry produced this finding, + // so a `update-contract` annotation can land on that entry's line rather + // than at the top of the file. + drifts.push({ + ...drift, + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + }); ruleDrifts++; } } @@ -593,6 +604,17 @@ function main() { fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); log(`wrote ${args.out}`); + // Emitted BEFORE the summary on purpose. GitHub renders annotations at the top + // of the run page, which is where a developer looks first; without them the only + // thing above the summary is "Process completed with exit code 1" and the + // natural next click goes to raw logs instead of the remediation. When the + // contract is part of the PR's diff these also attach inline to the exact + // expectation that drifted. + emitAnnotations(report, { + contractsDir: args.contracts, + workspace: process.env.GITHUB_WORKSPACE, + }); + const markdown = renderMarkdown(report, drifts, coverageHoles, legs); // eslint-disable-next-line no-console console.log(markdown); diff --git a/scripts/ppl-lint/annotate.mjs b/scripts/ppl-lint/annotate.mjs new file mode 100644 index 00000000000..06b1a384db7 --- /dev/null +++ b/scripts/ppl-lint/annotate.mjs @@ -0,0 +1,227 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * GitHub Actions annotations for the PPL lint multi-version check. + * + * The drift report and the job summary already say exactly what to change. The + * problem is WHERE a developer looks first: GitHub renders workflow-command + * annotations at the top of the run page and, when a finding names a file in the + * pull request, inline on that file in the Files-changed view. Without them the + * only thing above the summary is "Process completed with exit code 1", so the + * natural next click leads into raw job logs instead of the remediation. + * + * This module turns findings into `::error file=…,line=…::` commands. Two rules + * govern everything here: + * + * 1. An annotation must point at a line the reader can act on, or carry no line + * at all. A confidently wrong line number sends someone to edit the wrong + * expectation, which is worse than making them find it themselves. + * 2. The annotation is a POINTER, not the report. It carries the finding and the + * one-line action; the summary keeps the full reasoning. Annotation text is + * truncated by the UI, so front-load the identity of the problem. + * + * Inconclusive findings are deliberately `::warning`, not `::error`: they mean + * "we could not check", and the run is already red from the exit code. Rendering + * them as errors next to real drift would invite exactly the response the + * classifier works to prevent — editing a rule because a leg timed out. + */ + +import fs from 'fs'; +import path from 'path'; + +/** Escape a workflow-command property value (file/title). */ +function escapeProperty(value) { + return String(value) + .replace(/%/g, '%25') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A') + .replace(/:/g, '%3A') + .replace(/,/g, '%2C'); +} + +/** Escape a workflow-command message body; newlines must survive as %0A. */ +function escapeData(value) { + return String(value).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} + +/** + * Line of the `expectations[]` entry whose `version` is `range`, 1-indexed. + * + * Deliberately a text scan rather than a JSON walk: JSON.parse discards line + * information, and every consumer of this number is a human reading the file in a + * browser. Returns undefined when the range is absent or ambiguous (appears more + * than once), because an annotation with no line still lands on the file while a + * wrong line actively misleads. + */ +export function findExpectationLine(contractText, range) { + if (!contractText || !range) return undefined; + const lines = contractText.split('\n'); + const needle = `"version": ${JSON.stringify(range)}`; + const hits = []; + for (let i = 0; i < lines.length; i++) { + // Match on the normalized form so incidental whitespace does not defeat it. + if (lines[i].replace(/\s+/g, ' ').includes(needle)) hits.push(i + 1); + } + return hits.length === 1 ? hits[0] : undefined; +} + +/** + * Line of the top-level `"ruleId"` key, used as the fallback anchor when the + * finding is about the rule as a whole (a renamed grammar rule) rather than one + * pinned expectation. + */ +export function findRuleIdLine(contractText) { + if (!contractText) return undefined; + const lines = contractText.split('\n'); + for (let i = 0; i < lines.length; i++) { + if (/^\s*"ruleId"\s*:/.test(lines[i])) return i + 1; + } + return undefined; +} + +/** + * Repo-relative path of a contract file, for `file=`. + * + * GitHub only renders an annotation inline when the path is relative to the + * repository root AND the file is in the pull request's diff. `contractsDir` is + * an absolute path inside the workspace, so strip the workspace prefix. + */ +export function contractRepoPath(contractsDir, fileName, workspace) { + const absolute = path.join(contractsDir, fileName); + if (workspace && absolute.startsWith(workspace)) { + return path.relative(workspace, absolute); + } + return absolute; +} + +/** + * Build the annotation list for a drift report. Pure: returns descriptors so the + * caller decides where they are written and the tests can assert on them without + * capturing stdout. + */ +export function buildAnnotations(report, { contractsDir, workspace, readFile = readContract } = {}) { + const annotations = []; + const textCache = new Map(); + const contractText = (fileName) => { + if (!fileName) return undefined; + if (!textCache.has(fileName)) { + textCache.set(fileName, readFile(contractsDir, fileName)); + } + return textCache.get(fileName); + }; + + for (const drift of report.drifts || []) { + const file = drift.contractFile; + const text = contractText(file); + // `update-contract` findings are about one pinned expectation, so anchor + // there. Everything else is about the rule, so anchor at its identity line — + // the reader's next stop is the detector named in the message anyway. + const line = + (drift.expectationRange ? findExpectationLine(text, drift.expectationRange) : undefined) ?? + findRuleIdLine(text); + + annotations.push({ + level: drift.enforced ? 'error' : 'warning', + file: file ? contractRepoPath(contractsDir, file, workspace) : undefined, + line, + title: `PPL lint drift: ${drift.driftClass} (${drift.ruleId} @ ${drift.version})`, + // Message order matters: the UI truncates, so lead with what moved, then the + // action, then where. The summary carries the full rationale. + message: [ + drift.evidence, + `FIX (${drift.remediation.action}): ${drift.remediation.detail}`, + drift.query ? `QUERY: ${drift.query}` : undefined, + ] + .filter(Boolean) + .join('\n'), + }); + } + + for (const hole of report.coverageHoles || []) { + const text = contractText(hole.file); + annotations.push({ + level: hole.enforced ? 'error' : 'warning', + file: hole.file ? contractRepoPath(contractsDir, hole.file, workspace) : undefined, + line: findRuleIdLine(text), + title: `PPL lint coverage hole: ${hole.ruleId} @ ${hole.version}`, + message: + `No expectation in this contract matches engine ${hole.version}, so nothing pins ` + + `"${hole.ruleId}" there. Add a reviewed expectation whose version range covers ` + + `${hole.version}; do not widen an existing range to absorb it unless the behavior is ` + + `genuinely identical.`, + }); + } + + // Warning, not error: the linter is not what went wrong, and the run is already + // red from the exit code. See the module comment. + for (const entry of report.inconclusive || []) { + const text = contractText(entry.file); + annotations.push({ + level: 'warning', + file: entry.file ? contractRepoPath(contractsDir, entry.file, workspace) : undefined, + line: findRuleIdLine(text), + title: `PPL lint inconclusive: ${entry.ruleId} @ ${entry.version} (leg problem)`, + message: + `No case could be compared for "${entry.ruleId}" on engine ${entry.version}` + + (entry.reasons && entry.reasons.length > 0 ? ` — ${entry.reasons.join('; ')}` : '') + + `. This is NOT a lint finding: the engine or the detector run did not answer, so ` + + `nothing was validated. Check that leg's job logs and re-run. Do not edit the rule or ` + + `the contract on the strength of this.`, + }); + } + + for (const missing of report.missingContracts || []) { + const ruleId = missing.ruleId || missing; + annotations.push({ + level: 'error', + // A rule with no contract has no file to point at; the manifest is where the + // reader's edit goes. + file: undefined, + title: `PPL lint unvalidated rule: ${ruleId}`, + message: + `"${ruleId}" ships enabled at error severity in OSD's rules_catalog.json but ` + + `${missing.reason || 'has no contract in this corpus'}. A default-error rule with no ` + + `contract is invisible to this check. Add a contract file and list it under ` + + `manifest.defaultError, or lower the rule's severity in OSD.`, + }); + } + + return annotations; +} + +function readContract(contractsDir, fileName) { + try { + return fs.readFileSync(path.join(contractsDir, fileName), 'utf8'); + } catch { + // A contract we cannot read still deserves a file-less annotation. + return undefined; + } +} + +/** Render one descriptor as a workflow command line. */ +export function formatAnnotation(annotation) { + const props = []; + if (annotation.file) props.push(`file=${escapeProperty(annotation.file)}`); + if (annotation.line) props.push(`line=${annotation.line}`); + if (annotation.title) props.push(`title=${escapeProperty(annotation.title)}`); + const suffix = props.length > 0 ? ` ${props.join(',')}` : ''; + return `::${annotation.level}${suffix}::${escapeData(annotation.message)}`; +} + +/** + * Emit annotations for a report. No-op unless running under Actions (or forced), + * so a local run is not spammed with workflow-command noise. + */ +export function emitAnnotations(report, options = {}) { + const enabled = options.force || process.env.GITHUB_ACTIONS === 'true'; + if (!enabled) return []; + const annotations = buildAnnotations(report, options); + for (const annotation of annotations) { + // eslint-disable-next-line no-console + console.log(formatAnnotation(annotation)); + } + return annotations; +} From 4f9b599a921533fced255d502bbdd7e0cdba4c37 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 20:48:17 -0700 Subject: [PATCH 12/39] feat(ci): validate the compiled-simplified lint surface too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit OSD ships lint on two grammar surfaces and a user gets whichever one their session resolves to. This check only ever validated one of them. lintRuntimePPLQuery falls back to the compiled-simplified surface whenever the runtime bundle is unavailable — no dataset selected, an engine below 3.6, or a bundle that has not loaded. That path is not a degraded copy: field_validation runs a text-side pass keyed on grammarSurface === 'compiled-simplified' that the runtime path never executes. It is also the surface with no engine floor, since it needs no grammar export, so it is where old-engine coverage is possible at all (GET /_plugins/_ppl/_grammar landed in 3.6; 2.19 through 3.5 cannot export a bundle). Add PPL_LINT_SURFACE to the runner. It defaults to runtime-bundle, so the required check is unchanged, and the compiled surface is an explicit opt-in rather than a fallback: a missing bundle on the runtime surface stays a hard failure, because quietly linting OSD's own grammar instead of the candidate would validate the wrong thing. runtimeOnly rules are the trap. lint_runner skips them on the compiled surface because the productions they walk are absent there, so their zero diagnostics mean 'deliberately inert', not 'the detector went silent'. Counted as zero, three healthy rules would classify as detector-silent drift. They are now reported not-applicable, rendered 'n/a (surface)', and a rule whose every case is inert is n/a — not agree (it proved nothing) and not inconclusive (nothing went wrong, and there is nothing to re-run). Also key the summary matrix on the leg label instead of the engine version: two legs can share a version while validating different surfaces, and keying on version alone made them collide so one leg's cells rendered in place of the other's. Verified with a real compiled leg produced by the runner against a live OSD checkout, beside the real 3.7/pr-build runtime legs: baseline passes with the three runtimeOnly rules n/a; a regression injected only on the compiled surface is caught as detector-silent while both runtime legs stay green; the two same-version legs no longer collide; a report predating the surface field still reads as runtime-bundle. 16 existing drift scenarios and 59 unit tests unchanged. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/README.md | 49 +++++++- scripts/ppl-lint/aggregate-versions.mjs | 83 ++++++++++-- scripts/ppl-lint/run-frontend-contract.mjs | 139 +++++++++++++++++++-- 3 files changed, 247 insertions(+), 24 deletions(-) diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 0faebbb57ff..9dcc2c8d454 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -213,9 +213,52 @@ run the **same** contract oracle (`PplLintRuleValidationIT`) with against expectations — on an older engine a mismatch is the signal being collected, not a broken run. -**Engine floor: 3.6.0.** `GET /_plugins/_ppl/_grammar` landed in #5162, which is -an ancestor of 3.6 but not 3.5, so a 3.5 leg could not export a grammar bundle for -the detectors to lint against. +**Engine floor: 3.6.0 — for the runtime-bundle surface.** +`GET /_plugins/_ppl/_grammar` landed in #5162, which is an ancestor of 3.6 but not +3.5, so a 3.5 leg cannot export a grammar bundle for the detectors to lint against. + +### The two grammar surfaces + +OSD ships lint on **two** surfaces, and a user gets whichever one their session +resolves to (`lintRuntimePPLQuery`): + +| Surface | When the product uses it | Engine floor | +| --- | --- | --- | +| `runtime-bundle` | the engine exported a grammar bundle and it has loaded | 3.6.0 | +| `compiled-simplified` | no bundle — no dataset selected, engine below 3.6, or bundle not yet loaded | none | + +The compiled surface is not a degraded copy of the runtime one: it runs detector +logic the runtime path does not (`field_validation`'s text-side pass keys off +`grammarSurface === 'compiled-simplified'`). It is also the surface with no engine +floor, so it is where old-engine coverage is possible at all. + +`PPL_LINT_SURFACE` selects which surface a detector run validates. It defaults to +`runtime-bundle`, so the required check is unchanged, and the compiled surface is +an **explicit opt-in** — never a silent fallback. A missing bundle on the runtime +surface stays a hard failure, because quietly linting OSD's own grammar instead of +the candidate would validate the wrong thing. + +**`runtimeOnly` rules do not run on the compiled surface.** `lint_runner` skips +them (the productions they walk are absent from the compiled grammar), so a +compiled leg reports them `not-applicable` rather than as zero diagnostics. This +distinction is load-bearing: counted as zero, a healthy rule would classify as +`detector-silent` drift and send someone to "fix" it. In the summary table those +cells read `n/a (surface)`, and a rule whose every case is inert is `n/a` — not +`agree` (it proved nothing) and not `inconclusive` (nothing went wrong, and there +is nothing to re-run). + +Two legs may share an engine version while validating different surfaces, so the +matrix is keyed on the **leg label**, not the version. + +```bash +# A compiled-surface leg: no grammar bundle needed, so any engine version works. +PPL_LINT_SURFACE=compiled-simplified \ +PPL_LINT_CONTRACT_DIR= \ +PPL_LINT_TARGET_MANIFEST=/target.json \ +PPL_LINT_SCHEDULE=nightly \ +PPL_LINT_REPORT=/detector-report.json \ +node -r ./src/setup_node_env /scripts/ppl-lint/run-frontend-contract.mjs +``` This workflow is **non-enforcing for now**: it reports and uploads, while the required check stays the single-version `validation-result`. Promoting it needs a diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 83bb5bd39e1..18026bb095c 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -148,6 +148,9 @@ function loadLeg({ version, dir }) { label: version, dir, grammarHash: target.grammarHash || '', + // Which of OSD's two lint surfaces this leg validated. Older detector reports + // predate the field; they were all runtime-bundle runs. + surface: detector.surface || 'runtime-bundle', parserRuleNames: bundle && Array.isArray(bundle.parserRuleNames) ? bundle.parserRuleNames : undefined, detector, backend, @@ -386,6 +389,10 @@ function main() { // its engine verdicts or its detector rows). Tracked separately from drift // because the answer is "re-run / fix the leg", not "edit the linter". const inconclusive = []; + // Cases a leg's grammar surface cannot express (a `runtimeOnly` rule on a + // compiled-simplified leg). Recorded so the report can say WHY a cell is blank, + // but never a failure: the rule is inert there by design. + const notApplicable = []; const matrix = []; // one row per rule × version, for the summary table // A rule that ships enabled at error severity but has no contract file is @@ -420,7 +427,7 @@ function main() { }); if (grammarDrift) { drifts.push({ ...grammarDrift, enforced: isEnforced, contractFile: file }); - matrix.push({ ruleId, version: leg.version, status: 'drift', drifts: 1 }); + matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'drift', drifts: 1 }); continue; } } @@ -443,6 +450,7 @@ function main() { matrix.push({ ruleId, version: leg.version, + leg: leg.label, status: outOfScopeDrifts.length > 0 ? 'drift' : 'out-of-scope', drifts: outOfScopeDrifts.length, }); @@ -450,7 +458,7 @@ function main() { } // In scope on this engine but nothing pins its behavior there. coverageHoles.push({ ruleId, file, version: leg.version, enforced: isEnforced }); - matrix.push({ ruleId, version: leg.version, status: 'uncovered', drifts: 0 }); + matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'uncovered', drifts: 0 }); continue; } @@ -464,6 +472,10 @@ function main() { // otherwise render `agree` off the control alone. let triggersExpected = 0; let triggersCompared = 0; + // Cases this leg's surface cannot express. Counted separately from `unusable` + // because the two need opposite advice: not-applicable is expected and needs + // no action, unusable means something did not answer and needs a re-run. + let ruleNotApplicable = 0; const unusable = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; @@ -483,6 +495,23 @@ function main() { const detectorResult = (leg.detector.results || []).find( (r) => r.ruleId === ruleId && r.queryName === queryName ); + // A case the surface cannot express at all (a `runtimeOnly` rule on a + // compiled-simplified leg) is excluded rather than compared. Its zero + // diagnostics are `lint_runner` deliberately skipping the rule, so + // comparing them against a non-zero expectation would classify a healthy + // rule as detector-silent and send someone to fix it. This is NOT the same + // as `inconclusive`: nothing went wrong, and there is nothing to re-run. + if (detectorResult && detectorResult.notApplicable) { + notApplicable.push({ + ruleId, + version: leg.version, + queryName, + surface: leg.surface, + reason: detectorResult.notApplicable, + }); + ruleNotApplicable++; + continue; + } const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); const { observed, usable } = readBackendObservation(backendEntry, detectorResult); if (!usable) { @@ -541,7 +570,19 @@ function main() { // that lost every TRIGGER while keeping a control, since the triggers are // where the rule's behavior actually lives. Calling either agreement is the // vacuous pass this check exists to prevent. - if (compared === 0 || (triggersExpected > 0 && triggersCompared === 0)) { + // A rule the surface cannot express at all is `n/a`, not `inconclusive`: + // nothing failed and there is nothing to re-run, so it must not fail the run. + // Checked BEFORE the inconclusive test, which would otherwise catch it + // (compared === 0) and demand a re-run that could never change the outcome. + if (compared === 0 && ruleNotApplicable > 0) { + matrix.push({ + ruleId, + version: leg.version, + leg: leg.label, + status: 'not-applicable', + drifts: 0, + }); + } else if (compared === 0 || (triggersExpected > 0 && triggersCompared === 0)) { inconclusive.push({ ruleId, file, @@ -549,7 +590,7 @@ function main() { enforced: isEnforced, reasons: unusable, }); - matrix.push({ ruleId, version: leg.version, status: 'inconclusive', drifts: ruleDrifts }); + matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'inconclusive', drifts: ruleDrifts }); } else { if (unusable.length > 0) { log( @@ -560,6 +601,7 @@ function main() { matrix.push({ ruleId, version: leg.version, + leg: leg.label, status: ruleDrifts === 0 ? 'agree' : 'drift', drifts: ruleDrifts, }); @@ -577,6 +619,7 @@ function main() { label: l.label, engineVersion: l.version, grammarHash: l.grammarHash, + surface: l.surface, })), enforcedRules: [...enforcedRules].sort(), missingContracts, @@ -585,6 +628,7 @@ function main() { drifts, coverageHoles, inconclusive, + notApplicable, result: { driftCount: drifts.length, enforcedDriftCount: enforcedDrifts.length, @@ -662,25 +706,42 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { reasons.push(`${report.result.missingContractCount} unvalidated rule(s)`); } lines.push( - `Engine versions: ${legs.map((l) => `\`${l.version}\``).join(', ')} — ` + - `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` + // Name the surface when a leg is not the default runtime-bundle one, so a + // reader knows a column speaks for OSD's compiled grammar rather than the + // engine's exported one — the two do not run the same set of rules. + `Engine versions: ${legs + .map((l) => + l.surface && l.surface !== 'runtime-bundle' ? `\`${l.version}\` (${l.surface})` : `\`${l.version}\`` + ) + .join(', ')} — ` + `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` ); lines.push(''); - const versions = legs.map((l) => l.version); + // Columns are keyed on the LEG LABEL, not the engine version: two legs can share + // a version while validating different surfaces (a 3.7 runtime-bundle leg and a + // 3.7 compiled leg), and keying on version alone made them collide so one leg's + // results silently rendered in place of the other's. + const columns = legs.map((l) => ({ + label: l.label, + heading: + l.surface && l.surface !== 'runtime-bundle' + ? `\`${l.version}\`
${l.surface}` + : `\`${l.version}\``, + })); const rules = [...new Set(report.matrix.map((m) => m.ruleId))].sort(); - lines.push(`| Rule | ${versions.map((v) => `\`${v}\``).join(' | ')} |`); - lines.push(`| ---- | ${versions.map(() => '----').join(' | ')} |`); + lines.push(`| Rule | ${columns.map((c) => c.heading).join(' | ')} |`); + lines.push(`| ---- | ${columns.map(() => '----').join(' | ')} |`); const cell = { agree: 'agree', drift: 'DRIFT', uncovered: 'not covered', 'out-of-scope': 'n/a (out of scope)', + 'not-applicable': 'n/a (surface)', inconclusive: '**inconclusive**', }; for (const ruleId of rules) { - const cells = versions.map((version) => { - const row = report.matrix.find((m) => m.ruleId === ruleId && m.version === version); + const cells = columns.map((column) => { + const row = report.matrix.find((m) => m.ruleId === ruleId && m.leg === column.label); if (!row) return '—'; if (row.status === 'drift') return `**DRIFT** (${row.drifts})`; // An unmapped status must still render as something visible. A blank cell diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index e45b4a48b04..905a44e41ae 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -45,6 +45,33 @@ * — a trigger the detector flags is one the backend rejected; a control the * detector passes is one the backend accepted (design §3.2, §4.3). * 4. Coverage (nightly only): every enabled catalog rule has a contract file. + * + * ## Two grammar surfaces + * + * OSD ships lint on TWO surfaces, and a user gets whichever one their session + * resolves to: + * + * runtime-bundle the candidate grammar the engine exported. Requires + * `GET /_plugins/_ppl/_grammar`, which landed in 3.6. + * compiled-simplified OSD's own checked-in grammar, used whenever the runtime + * bundle is unavailable — no dataset selected, an engine + * below 3.6, or a bundle that has not loaded yet. This is + * `lintRuntimePPLQuery`'s fallback path, and it runs + * detector logic the runtime path does not (see + * field_validation's text-side pass). + * + * `PPL_LINT_SURFACE` selects which one this run validates; it defaults to + * `runtime-bundle`, so the required check is unchanged. The compiled surface is + * an EXPLICIT opt-in, never a silent fallback: the whole point of the required + * check is that a missing bundle is a hard failure rather than a quiet + * downgrade to OSD's own grammar (which would validate the wrong thing). + * + * The compiled surface is what makes pre-3.6 engine legs meaningful. It also + * carries a mandatory caveat: `runtimeOnly` rules (multisearch/union/replace + * arity) are SKIPPED on it by `lint_runner` because the productions they walk do + * not exist in the compiled grammar. A compiled leg therefore reports them as + * `not-applicable` rather than as zero diagnostics, so the aggregator cannot + * mistake a deliberately-inert rule for a detector that regressed. */ import fs from 'fs'; @@ -54,11 +81,39 @@ import { createRequire } from 'module'; // OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved // against the OSD checkout root, not this script's SQL-repo location. const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; +// The COMPILED-simplified surface: OSD's own checked-in grammar, used when the +// engine cannot export a runtime bundle. See `PPL_LINT_SURFACE` below. +const ANALYZER_MODULE = 'packages/osd-monaco/src/ppl/ppl_language_analyzer'; // The Monaco-free engine barrel (@osd/monaco/ppl-lint) exposes the catalog; the // detector registry is a deep import used only for the wiring registration check. const CATALOG_MODULE = 'packages/osd-monaco/ppl-lint'; +// Source-path fallback for the catalog. The `ppl-lint` subpath is a built export +// that only exists on checkouts that ship it; the compiled surface deliberately +// supports older checkouts (that is the coverage it adds), so fall back to the +// source module, which `setup_node_env` transpiles on require anyway. +const CATALOG_SOURCE_MODULE = 'packages/osd-monaco/src/ppl/lint/catalog'; const DETECTOR_REGISTRY_MODULE = 'packages/osd-monaco/target/ppl/lint/detector_registry.js'; +/** + * Which grammar surface this run validates. Defaults to `runtime-bundle` so the + * required check's behavior is unchanged; `compiled-simplified` is an explicit + * opt-in for legs whose engine cannot export a bundle. + */ +const SURFACE = (() => { + const requested = process.env.PPL_LINT_SURFACE || 'runtime-bundle'; + if (requested !== 'runtime-bundle' && requested !== 'compiled-simplified') { + // A typo must not silently select the default: that would report compiled + // results under a runtime-bundle label, or vice versa. + // eslint-disable-next-line no-console + console.error( + `[ppl-lint-frontend] FATAL: PPL_LINT_SURFACE must be "runtime-bundle" or ` + + `"compiled-simplified", got "${requested}".` + ); + process.exit(2); + } + return requested; +})(); + function log(message) { // eslint-disable-next-line no-console console.log(`[ppl-lint-detector-contract] ${message}`); @@ -143,10 +198,41 @@ function loadOsd() { } }; - const headless = resolveOsd(HEADLESS_MODULE); - const { getBundledCatalog } = resolveOsd(CATALOG_MODULE); + // Prefer the built subpath (what the required check uses); fall back to source + // so a checkout without the built export can still run the compiled surface. + const catalogModule = + resolveOsd(CATALOG_MODULE, { optional: true }) || resolveOsd(CATALOG_SOURCE_MODULE); + const { getBundledCatalog } = catalogModule; const registry = resolveOsd(DETECTOR_REGISTRY_MODULE, { optional: true }); + if (typeof getBundledCatalog !== 'function') { + fatal(`getBundledCatalog not found in ${CATALOG_MODULE} or ${CATALOG_SOURCE_MODULE}.`); + } + const getDetector = registry && registry.getDetector; + + // On the compiled surface the headless bundle API is not needed at all, and + // requiring it would make this mode unusable on an OSD checkout that predates + // it — precisely the older-version coverage the mode exists to provide. + if (SURFACE === 'compiled-simplified') { + const { PPLLanguageAnalyzer } = resolveOsd(ANALYZER_MODULE); + if (typeof PPLLanguageAnalyzer !== 'function') { + fatal(`PPLLanguageAnalyzer not found in ${ANALYZER_MODULE}.`); + } + const analyzer = new PPLLanguageAnalyzer(); + return { + surface: SURFACE, + // Same (query, grammar, context) shape as the bundle path so the main loop + // does not branch per surface; `grammar` is unused here. + lintQuery: (query, _grammar, context) => { + const analysis = analyzer.analyzeLint(query, context); + return (analysis && analysis.result) || { diagnostics: [] }; + }, + getBundledCatalog, + getDetector, + osdRoot, + }; + } + const headless = resolveOsd(HEADLESS_MODULE); const { deserializeBundleOrThrow, lintQueryWithBundle } = headless; if (typeof deserializeBundleOrThrow !== 'function' || typeof lintQueryWithBundle !== 'function') { fatal( @@ -155,12 +241,15 @@ function loadOsd() { `Is the OSD checkout on a branch that ships the headless API (design §4.3)?` ); } - if (typeof getBundledCatalog !== 'function') { - fatal(`getBundledCatalog not found in ${CATALOG_MODULE}.`); - } - const getDetector = registry && registry.getDetector; - return { deserializeBundleOrThrow, lintQueryWithBundle, getBundledCatalog, getDetector, osdRoot }; + return { + surface: SURFACE, + deserializeBundleOrThrow, + lintQuery: lintQueryWithBundle, + getBundledCatalog, + getDetector, + osdRoot, + }; } /** Load the candidate grammar bundle + deserialize it once (fail loud; CI has no fallback). */ @@ -403,10 +492,13 @@ function main() { const reportPath = process.env.PPL_LINT_REPORT; const osd = loadOsd(); - const { getBundledCatalog, getDetector, lintQueryWithBundle, osdRoot } = osd; + const { getBundledCatalog, getDetector, lintQuery, osdRoot, surface } = osd; const catalog = getBundledCatalog(); - const grammar = loadCandidateGrammar(osd); + // The compiled surface lints with OSD's own checked-in grammar, so there is no + // candidate bundle to load. On the runtime surface a missing bundle stays a hard + // failure — never a quiet downgrade to the compiled grammar. + const grammar = surface === 'compiled-simplified' ? undefined : loadCandidateGrammar(osd); const target = loadTarget(); const engineVersion = target.engineVersion || process.env.PPL_SQL_VERSION || ''; const backendReport = loadBackendReport(); @@ -417,6 +509,11 @@ function main() { osdRoot, schedule, engineVersion, + // Which of OSD's two lint surfaces produced these results. The aggregator + // needs this to interpret them: a compiled leg legitimately has no verdict for + // `runtimeOnly` rules, and mixing the two surfaces under one label would + // report a deliberately-inert rule as a regression. + surface, grammarHash: target.grammarHash || '', differential: !!backendReport, // Census of the rules that ship enabled at ERROR severity, read from the OSD @@ -474,7 +571,29 @@ function main() { const expected = expectedQueries[queryName]; const expectedCount = expected.detectorCount; - const result = lintQueryWithBundle(query, grammar, context); + // A `runtimeOnly` rule walks grammar productions that exist only in the + // runtime bundle, so `lint_runner` skips it on the compiled surface. Its + // zero diagnostics here mean "deliberately inert", NOT "the detector went + // silent" — reporting them as a count would make the aggregator classify a + // healthy rule as detector-silent drift and send someone to fix it. Mark the + // case not-applicable and let the aggregator exclude it. + if (surface === 'compiled-simplified' && entry && entry.runtimeOnly) { + log( + ` SKIP ${ruleId}/${queryName} (${role}): runtimeOnly rule is inert on the ` + + `compiled-simplified surface.` + ); + report.results.push({ + ruleId, + queryName, + role, + query, + surface, + notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', + }); + continue; + } + + const result = lintQuery(query, grammar, context); const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); const actual = matches.length; const ok = actual === expectedCount; From 0050f837fb5803f35e0a8932d373fe8a828ac1c7 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 21:07:08 -0700 Subject: [PATCH 13/39] feat(ci): add compiled-surface legs for engines below the grammar floor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the multi-version matrix down to 2.19 by validating the surface that actually reaches those users. Engines below 3.6 cannot export a grammar bundle, so the runtime-bundle surface cannot reach them at all. The compiled-simplified surface has no such floor — it is what a user gets whenever no bundle is available, which on a pre-3.6 cluster is always. Those legs still run the real contract queries against the real engine, so the backend half of the differential is genuine. Workflow: a nightly observe-compiled matrix (2.19.0 / 3.0.0 / 3.5.0, overridable via compiled_versions, [] to skip; pull_request skips by default since three more engine images is too slow per PR). Its observe job omits -Dppl.lint.grammar.bundle, so the IT exports nothing, and writes a marker — which is what distinguishes 'this engine has no _grammar endpoint' from 'the bundle export failed', a distinction that must stay a hard error. Contracts now declare the surface(s) they were verified against and are only scored on a matching leg. This turns grammarSurface from a decorative field into a real guard: judged on a surface it never claimed, a contract produces confident nonsense — a runtime-bundle contract on a compiled leg yielded both version-scope-too-narrow and a coverage hole, each about a surface it does not describe. Four cross-surface contracts are marked 'both' after verifying their grammar is byte-identical 2.19 through 3.7 (headCommand, dedupCommand+CONSECUTIVE, joinType's full alternative list, evalCommand). field-validation and unsupported-window-function-in-eventstats were also marked 'both': their previous 'compiled-simplified' described where the rule CAN run, not a restriction, and honoring it as exclusive would have dropped them from every runtime leg including the required check's. Live-verified on the real 3.7 runtime leg that both still fire exactly as pinned. field-validation gains a <3.4.0 expectation, since its empty appliesTo ships it to users on every engine. Detector behavior is live-verified identical at 2.19.0, 3.0.0, 3.5.0 and 3.7.0. Its backend oracle deliberately omits error.type/reason — that wording has not been observed live on those engines, and inventing one would either fail spuriously or get 'fixed' by pinning whatever CI first happened to see. assertRejection now tolerates an omitted error block so 'it rejects' can be asserted without claiming to know how. Found and fixed while wiring this up: division-by-zero pinned modulo-by-zero-flagged at detectorCount 1, but the detector deliberately handles only '/' (division_by_zero.ts: 'Modulo-by-zero was not verified live'). The contract asserted behavior the rule never had; nightly-only with no compiled leg meant it had never run. It is now a control documenting that boundary. Verified against a 4-leg mixed-surface matrix built from the real leg artifacts: PASS with 2.19 correctly reporting 5 rules n/a (surface), eventstats n/a (out of scope), and 4 rules genuinely validated. 5 new surface scenarios, 16 existing drift scenarios and 59 unit tests all green. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 176 +++++++++++++++++- .../remote/PplLintRuleValidationIT.java | 17 +- .../dedup-consecutive-unsupported.spec.json | 2 +- .../contracts/disabled-join-type.spec.json | 2 +- .../contracts/division-by-zero.spec.json | 12 +- .../contracts/field-validation.spec.json | 40 +++- .../contracts/head-without-sort.spec.json | 2 +- ...ed-window-function-in-eventstats.spec.json | 2 +- scripts/ppl-lint/README.md | 15 ++ scripts/ppl-lint/aggregate-versions.mjs | 27 +++ scripts/ppl-lint/run-frontend-contract.mjs | 35 ++++ 11 files changed, 308 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index e9ee6849111..f27079ba1f2 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -72,14 +72,32 @@ on: description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' required: false type: string + compiled_versions: + description: 'JSON array of engine versions to validate on the compiled-simplified surface, e.g. ["2.19.0"]. Use "[]" to skip them.' + required: false + type: string permissions: contents: read env: - # Released engine versions to validate, newest last. Each must be >= 3.6.0 (the - # _grammar endpoint floor) and must have a published distribution image. + # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must + # be >= 3.6.0 (the _grammar endpoint floor) and must have a published + # distribution image. ENGINE_VERSIONS: '["3.6.0","3.7.0"]' + # Released engine versions to validate on the COMPILED-SIMPLIFIED surface. + # + # These engines cannot export a grammar bundle (GET /_plugins/_ppl/_grammar + # landed in 3.6), so the runtime surface cannot reach them at all. But the + # compiled surface has no such floor: it lints with OSD's own checked-in grammar, + # which is exactly what a user gets when no bundle is available — including every + # user on an engine below 3.6. Those legs still run the real contract queries + # against the real engine, so the backend half of the differential is genuine. + # + # Only contracts declaring `grammarSurface: "both"` are scored here; the rest are + # reported not-applicable. Nightly only — see the `compiled_versions` input to + # run one ad hoc. + COMPILED_ENGINE_VERSIONS: '["2.19.0","3.0.0","3.5.0"]' jobs: # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a @@ -96,6 +114,7 @@ jobs: runs-on: ubuntu-latest outputs: released: ${{ steps.plan.outputs.released }} + compiled: ${{ steps.plan.outputs.compiled }} osd_repo: ${{ steps.plan.outputs.osd_repo }} osd_ref: ${{ steps.plan.outputs.osd_ref }} steps: @@ -104,6 +123,9 @@ jobs: env: REQUESTED_VERSIONS: ${{ inputs.engine_versions }} DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} + REQUESTED_COMPILED: ${{ inputs.compiled_versions }} + DEFAULT_COMPILED: ${{ env.COMPILED_ENGINE_VERSIONS }} + EVENT_NAME: ${{ github.event_name }} REQUESTED_REPO: ${{ inputs.osd_repo }} REQUESTED_REF: ${{ inputs.osd_ref }} VAR_REPO: ${{ vars.OSD_REPO }} @@ -121,6 +143,28 @@ jobs: assert isinstance(item,str), 'engine_versions entries must be strings' " echo "released=$released" >> "$GITHUB_OUTPUT" + + # Compiled-surface legs add three more engine images, so they run on the + # nightly schedule (and on an explicit dispatch), not on every PR that + # touches the corpus. An explicit input always wins, including "[]". + if [ -n "${REQUESTED_COMPILED:-}" ]; then + compiled="$REQUESTED_COMPILED" + elif [ "$EVENT_NAME" = "pull_request" ]; then + compiled='[]' + else + compiled="$DEFAULT_COMPILED" + fi + # An EMPTY list is legitimate here (unlike engine_versions): it means "skip + # the compiled surface this run". Still reject a non-list. + echo "$compiled" | python3 -c " + import json,sys + v=json.load(sys.stdin) + assert isinstance(v,list), 'compiled_versions must be a JSON array' + for item in v: + assert isinstance(item,str), 'compiled_versions entries must be strings' + " + echo "compiled=$compiled" >> "$GITHUB_OUTPUT" + echo "Compiled-surface legs: $compiled" >> "$GITHUB_STEP_SUMMARY" # Same precedence as the sibling workflow: dispatch input, then repo # variable, then the canonical upstream default. echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" @@ -223,6 +267,103 @@ jobs: name: ppl-lint-leg-${{ matrix.version }}-logs path: integ-test/build/reports/** + # Legs for engines BELOW the _grammar endpoint floor (3.6). These cannot export a + # grammar bundle, so they are observed for backend behavior only and their + # detector pass runs on the compiled-simplified surface — which is what a real + # user on such an engine gets, since no bundle can ever load there. + # + # Identical to observe-released except that `-Dppl.lint.grammar.bundle` is + # omitted: the IT skips the export when that property is unset, so no bundle + # fetch is attempted against an engine that has no such endpoint. + observe-compiled: + name: Observe engine ${{ matrix.version }} (compiled surface) + needs: plan + # An empty compiled list means "skip this surface" (the pull_request default). + if: ${{ needs.plan.outputs.compiled != '[]' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.compiled) }} + services: + opensearch: + image: opensearchproject/opensearch:${{ matrix.version }} + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Wait for the engine and confirm its version + id: engine + run: | + set -euo pipefail + for i in $(seq 1 40); do + if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi + echo "waiting for engine (${i}/40)..." + sleep 5 + done + cat /tmp/root.json + reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") + case "$reported" in + ${{ matrix.version }}*) ;; + *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + esac + curl -sf http://localhost:9200/_cat/plugins | grep -i sql + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + - name: Run contract observation against engine ${{ matrix.version }} + run: | + set -euo pipefail + mkdir -p leg + # No -Dppl.lint.grammar.bundle: this engine predates the _grammar endpoint, + # and the IT correctly exports nothing when the property is unset. + ./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ + -Dppl.lint.target="$(pwd)/leg/target.json" + # Mark the leg so the detect job knows to lint it on the compiled surface. + # A leg with no bundle would otherwise look like a failed export. + echo 'compiled-simplified' > leg/surface + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-${{ matrix.version }}-compiled + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-${{ matrix.version }}-compiled-logs + path: integ-test/build/reports/** + # The PR's own engine build, so the newest point in the matrix is the code under # review rather than the last release. Same oracle as the released legs; the only # difference is a Gradle-managed cluster instead of a published image, which is @@ -294,6 +435,7 @@ jobs: needs: - plan - observe-released + - observe-compiled - observe-pr-build if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest @@ -369,13 +511,25 @@ jobs: exit 1 fi for leg in "${legs[@]}"; do - # Skip the log-only artifacts an observation failure may have uploaded. - [ -f "$leg/ppl-grammar-bundle.json" ] || { echo "skipping $leg (no grammar bundle)"; continue; } version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') - echo "=== detectors vs engine $version ===" + # A leg is compiled-surface when its observe job said so. That marker is + # what distinguishes "this engine has no _grammar endpoint" from "the + # bundle export failed", which must stay a hard error. + if [ -f "$leg/surface" ] && [ "$(cat "$leg/surface")" = 'compiled-simplified' ]; then + surface_env=(PPL_LINT_SURFACE=compiled-simplified) + echo "=== detectors vs engine $version (compiled-simplified surface) ===" + elif [ -f "$leg/ppl-grammar-bundle.json" ]; then + surface_env=(PPL_LINT_SURFACE=runtime-bundle + PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json") + echo "=== detectors vs engine $version (runtime-bundle surface) ===" + else + # Skip the log-only artifacts an observation failure may have uploaded. + echo "skipping $leg (no grammar bundle and no compiled-surface marker)" + continue + fi + env "${surface_env[@]}" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ PPL_LINT_SCHEDULE=nightly \ - PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json" \ PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ PPL_LINT_REPORT="$leg/detector-report.json" \ node -r ./src/setup_node_env \ @@ -399,6 +553,7 @@ jobs: id: aggregate env: RELEASED: ${{ needs.plan.outputs.released }} + COMPILED: ${{ needs.plan.outputs.compiled }} run: | set -euo pipefail shopt -s nullglob @@ -419,7 +574,14 @@ jobs: # a matrix that silently lost one — the exact vacuous pass this workflow # exists to prevent. A dead leg is a failure, not a smaller matrix. missing=() - for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") pr-build; do + # Compiled legs are labelled "-compiled" to match their artifact + # name, so they occupy their own column even when a runtime leg validated + # the same engine version. + compiled_wanted=$(echo "$COMPILED" | python3 -c " + import json,sys + print(' '.join(f'{v}-compiled' for v in json.load(sys.stdin))) + ") + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build; do found=no for have in "${present[@]}"; do [ "$have" = "$want" ] && found=yes && break diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 7ae505e9b35..0a0b686cc44 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -419,12 +419,21 @@ private void assertRejection( expectedBody.getInt("status"), obs.body.getInt("status")); + // A contract may omit `error` entirely to assert only THAT the engine rejects, + // without pinning wording that has not been observed live on that version. That + // is weaker than a full oracle but honest; inventing a type/reason would either + // fail spuriously or get "fixed" by pinning whatever CI first happened to see. + if (!expectedBody.has("error")) { + return; + } JSONObject expectedError = expectedBody.getJSONObject("error"); JSONObject actualError = obs.body.getJSONObject("error"); - assertEquals( - "case \"" + queryName + "\": unexpected error.type for query: " + query, - expectedError.getString("type"), - actualError.getString("type")); + if (expectedError.has("type")) { + assertEquals( + "case \"" + queryName + "\": unexpected error.type for query: " + query, + expectedError.getString("type"), + actualError.getString("type")); + } if (expectedError.has("reason")) { assertEquals( "case \"" + queryName + "\": unexpected error.reason for query: " + query, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index ca414f6f103..90a614307b7 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "dedup-consecutive-unsupported", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "dedup-consecutive-unsupported", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json index bd0769934ea..25cd3232e88 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "disabled-join-type", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "disabled-join-type", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index d9071cc4978..5e1b33884c8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,7 +1,8 @@ { "schemaVersion": 3, "ruleId": "division-by-zero", - "grammarSurface": "compiled-simplified", + "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL — it documents that boundary rather than asserting a gap.", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "division-by-zero", @@ -29,8 +30,8 @@ "role": "control", "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" }, - "modulo-by-zero-flagged": { - "role": "trigger", + "modulo-by-zero-not-flagged": { + "role": "control", "query": "source={{index}} | eval m = balance % 0 | fields m | head 1" } }, @@ -47,9 +48,8 @@ "detectorCount": 0, "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } }, - "modulo-by-zero-flagged": { - "detectorCount": 1, - "severity": "warning", + "modulo-by-zero-not-flagged": { + "detectorCount": 0, "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index 45f85e3b769..984fcf851c1 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "field-validation", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "field-validation", @@ -56,6 +56,44 @@ } }, "expectations": [ + { + "version": "<3.4.0", + "note": "This rule has an empty appliesTo, so it ships to users on EVERY engine, including pre-3.4. Detector behavior is live-verified identical from 2.19 up (1/1/0 on the compiled surface at 2.19.0, 3.0.0, 3.5.0, 3.7.0). The backend oracle deliberately omits error.type/reason: this engine's wording for an unknown field has not been observed live, and inventing one would either fail spuriously or get 'fixed' by pinning whatever CI first happened to see. A compiled-surface leg records the real wording; pin it then.", + "queries": { + "unknown-field-existence": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + }, + "grok-field-slot-shape-typo": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400 + } + } + }, + "known-field-control": { + "detectorCount": 0, + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + }, { "version": ">=3.4.0 <3.7.0", "queries": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index e86a19c1002..695f1b6b550 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "head-without-sort", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "nightly", "wiring": { "detector": "head-without-sort", diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index d6821273175..4f436a383cb 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "unsupported-window-function-in-eventstats", "detectorPath": "packages/osd-monaco/src/ppl/lint/rules/unsupported_window_function.ts", - "grammarSurface": "compiled-simplified", + "grammarSurface": "both", "schedule": "pr", "wiring": { "detector": "unsupported-window-function-in-eventstats", diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 9dcc2c8d454..f20633768d4 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -250,6 +250,21 @@ is nothing to re-run). Two legs may share an engine version while validating different surfaces, so the matrix is keyed on the **leg label**, not the version. +Each contract declares the surface(s) it was verified against, and a contract is +only scored on a matching leg — `"both"` opts into either. Judged on a surface it +never claimed, every verdict is meaningless: a runtime-bundle contract on a +compiled leg yields both `version-scope-too-narrow` ("the engine rejects but the +rule is scoped away") and a coverage hole, each about a surface the contract does +not describe. Contracts declaring `"both"` are what a pre-3.6 leg can actually +validate; the rest report `n/a (surface)`. + +**Compiled-surface legs run nightly** (`COMPILED_ENGINE_VERSIONS`, default +`2.19.0` / `3.0.0` / `3.5.0`) — three more engine images is too slow for every PR. +Dispatch with `compiled_versions` to run one ad hoc, or `[]` to skip. Their observe +job omits `-Dppl.lint.grammar.bundle` (the IT then exports nothing) and writes a +`surface` marker file, which is what tells the detect job to lint them on the +compiled surface rather than treating a missing bundle as a failed export. + ```bash # A compiled-surface leg: no grammar bundle needed, so any engine version works. PPL_LINT_SURFACE=compiled-simplified \ diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index 18026bb095c..a0ab0f69613 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -411,6 +411,33 @@ function main() { const appliesTo = (spec.wiring && spec.wiring.appliesTo) || {}; for (const leg of legs) { + // A contract only speaks for the surface(s) it declares. Judge it on any + // other leg and every verdict is meaningless: a runtime-bundle contract on a + // compiled leg yields "the engine rejects but the rule is scoped away" + // (version-scope-too-narrow) and "no expectation covers this engine" + // (coverage hole) — both about a surface the contract never claimed to + // describe. Checked FIRST, before scope, grammar and coverage, because all + // three of those produce confident findings from an irrelevant comparison. + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + const legSurface = leg.surface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== legSurface) { + notApplicable.push({ + ruleId, + version: leg.version, + leg: leg.label, + surface: legSurface, + reason: `contract declares grammarSurface "${contractSurface}"`, + }); + matrix.push({ + ruleId, + version: leg.version, + leg: leg.label, + status: 'not-applicable', + drifts: 0, + }); + continue; + } + const inScope = versionInAppliesTo(appliesTo, leg.version); // A parser rule that vanished from the grammar is one fact about this diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 905a44e41ae..6f7ecd6de50 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -505,6 +505,9 @@ function main() { const contracts = loadContracts(); const failures = []; + // Contracts this surface did not score, recorded so the report says a rule was + // skipped for surface rather than leaving its absence unexplained. + const skippedForSurface = []; const report = { osdRoot, schedule, @@ -514,6 +517,9 @@ function main() { // `runtimeOnly` rules, and mixing the two surfaces under one label would // report a deliberately-inert rule as a regression. surface, + // Contracts whose declared `grammarSurface` excludes this run, so a reader can + // see WHY a rule has no scored cases here. + skippedForSurface, grammarHash: target.grammarHash || '', differential: !!backendReport, // Census of the rules that ship enabled at ERROR severity, read from the OSD @@ -547,6 +553,35 @@ function main() { continue; } + // A contract declares the surface its expectations were verified against. + // Until now that field was decorative; honoring it keeps a runtime-bundle + // contract from being scored on a compiled leg, where its rule may legitimately + // behave differently. `both` opts into being checked on either surface. + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface !== 'both' && contractSurface !== surface) { + log( + `SKIP ${ruleId} (grammarSurface=${contractSurface}, running ${surface}) — ` + + `${path.basename(file)}` + ); + skippedForSurface.push({ ruleId, contractSurface }); + // Emit an explicit not-applicable row per query rather than dropping the rule. + // Dropping it leaves the aggregator with no rows at all, which it correctly + // reads as `inconclusive` — "we could not check" — and fails on. But nothing + // went wrong here and there is nothing to re-run: this contract simply does + // not describe this surface. + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + report.results.push({ + ruleId, + queryName, + role: queryDef.role || 'trigger', + query: (queryDef.query || '').split('{{index}}').join(index), + surface, + notApplicable: `contract declares grammarSurface "${contractSurface}"`, + }); + } + continue; + } + const entry = checkWiring(spec, catalog, getDetector, failures); if (!entry) { continue; From 152fc2c0bb8ce8ff0fc896795d1eefe540884ef4 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Sun, 26 Jul 2026 21:36:00 -0700 Subject: [PATCH 14/39] fix(ci): stop the index wipe racing plugin initialization The 2.19 observation leg failed with a 60s SocketTimeoutException before any contract query ran. Root cause, from the engine's own logs: 04:16:04 MetadataCreateIndexService [.plugins-ml-config] creating index 04:16:05 MLSyncUpCron ML configuration initialized ~04:16:38 test client DELETE /.plugins-ml-config -> blocks 04:17:38 test SocketTimeoutException: 60000 MILLISECONDS The IT's first act is cleanUpIndices -> wipeAllOpenSearchIndices, which lists every index including hidden ones and DELETEs anything not matching .opensearch / .opendistro / .ql. The index ".plugins-ml-config" matches none of those, so it gets deleted -- and on an engine still initializing it, that request never completes. Why only 2.19: the cluster-health API reports GREEN before bundled plugins finish creating their system indices. The 3.7 container settled ~11s sooner, so its wipe landed 154s after ML init finished and succeeded. Same code, same args, different timing -- the 3.7 leg passed by luck, not by design. Two fixes, because either alone leaves a window: - Never delete indices under the ".plugins-" prefix. They belong to bundled plugins, wiping them is never the point of a test, and doing it during initialization hangs the suite. - Wait for the index set to stop changing before starting the IT (three identical consecutive listings), so a slow engine cannot race the wipe at all. Warns rather than fails if it does not settle: a slow-but-working engine should still be observed, and a genuinely unreachable one fails loudly in the next step. Also corrects an earlier claim of mine: the Gradle-provisioned cluster visible in that leg's log (build/testclusters/integTestRemote-0) started 7 seconds AFTER the failure was reported, so it was a consequence, not the cause. The released legs do reach their own engines -- their target.json files report three distinct versions and three distinct grammar hashes. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 34 +++++++++++++++++++ .../sql/legacy/OpenSearchSQLRestTestCase.java | 12 ++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index f27079ba1f2..4d3e9797bc3 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -323,6 +323,40 @@ jobs: esac curl -sf http://localhost:9200/_cat/plugins | grep -i sql + # `_cluster/health` goes GREEN before the bundled plugins finish creating + # their system indices, and the IT's first act is to wipe every non-system + # index. On 2.19 that DELETE landed while ML Commons was still initializing + # `.plugins-ml-config` and blocked until the client's 60s socket timeout, + # failing the leg before a single contract query ran. Wait for the plugin + # indices to stop appearing, so the wipe cannot race initialization. + - name: Wait for bundled plugin system indices to settle + run: | + set -euo pipefail + previous="" + stable=0 + for i in $(seq 1 30); do + current=$(curl -sf "http://localhost:9200/_cat/indices?h=index&expand_wildcards=all" \ + | sort | tr '\n' ',' || true) + if [ -n "$current" ] && [ "$current" = "$previous" ]; then + stable=$((stable + 1)) + # Three consecutive identical listings: no plugin is still creating + # indices. One match is not enough — initialization has gaps between + # an index being created and the next one starting. + if [ "$stable" -ge 3 ]; then + echo "index set stable after ${i} poll(s): $current" + exit 0 + fi + else + stable=0 + fi + previous="$current" + sleep 2 + done + # Not fatal: a slow-but-working engine should still be observed. The IT + # tolerates a wipe failure per index, and a genuinely unreachable cluster + # fails loudly in the next step anyway. + echo "::warning::plugin index set did not stabilize; continuing" + - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 91584fb45cf..267adea43c3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -213,9 +213,19 @@ protected static void wipeAllOpenSearchIndices(RestClient client) throws IOExcep String indexName = jsonObject.getString("index"); try { // System index, mostly named .opensearch-xxx or .opendistro-xxx, are not allowed to - // delete + // delete. + // + // `.plugins-` covers the system indices of bundled plugins (ML Commons' + // `.plugins-ml-config`, and friends). Deleting those is never the point of a + // test wipe, and it is actively harmful: on an engine whose plugins are + // still initializing, the DELETE blocks until the client's socket timeout + // and fails the suite before any test runs. That is what broke the 2.19 + // observation leg of the PPL lint multi-version matrix — `_cluster/health` + // reports GREEN before ML Commons finishes creating its config index, so + // the wipe raced initialization. if (!indexName.startsWith(".opensearch") && !indexName.startsWith(".opendistro") + && !indexName.startsWith(".plugins-") && !indexName.startsWith(".ql")) { client.performRequest(new Request("DELETE", "/" + indexName)); } From ead9ae0efd126c8c00066fa67142850ebe72a595 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 08:37:54 -0700 Subject: [PATCH 15/39] test(ci): probe the framework's startup requests on a compiled leg The 2.19 leg still times out after the settle fix, and a bare SocketTimeoutException does not say WHICH request hung. The stack points at OpenSearchRestTestCase.initClient line 216, which is GET _nodes/plugins -- a node-level API, unlike the _cat/plugins call the health step already makes successfully. Time each of the framework's startup requests explicitly so the next run identifies the culprit instead of inviting another guess. Diagnostic only; warns rather than fails. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 4d3e9797bc3..d0d18eca2fb 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -357,6 +357,24 @@ jobs: # fails loudly in the next step anyway. echo "::warning::plugin index set did not stabilize; continuing" + # Probe the EXACT requests the test framework makes before any test runs. + # `OpenSearchRestTestCase.initClient` issues `GET _nodes/plugins`, and + # `wipeAllOpenSearchIndices` issues `GET _cat/indices?expand_wildcards=all`. + # A leg that dies with a bare socket timeout gives no clue which of those + # hung, so time them here where the output is readable. + - name: Probe the framework's own startup requests + run: | + set -uo pipefail + for path in "_nodes/plugins" "_cat/indices?format=json&expand_wildcards=all" "_cluster/health"; do + start=$(date +%s) + if curl -sS --max-time 30 -o /tmp/probe.out -w '%{http_code}' \ + "http://localhost:9200/${path}" > /tmp/probe.code 2>/tmp/probe.err; then + echo "OK $(($(date +%s) - start))s HTTP $(cat /tmp/probe.code) ${path} ($(wc -c < /tmp/probe.out) bytes)" + else + echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" + fi + done + - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 with: From f7f4e8be33ee25ac1ebd6d40a9fddaf0e47168df Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 08:51:35 -0700 Subject: [PATCH 16/39] test(ci): probe HTTP/2 negotiation on a pre-3.x engine The first probe showed all three of the framework's startup requests returning HTTP 200 in 0s over curl, while the test JVM still times out on _nodes/plugins. So the endpoint works and the address is right; the remaining difference is how the client connects. RestClientBuilder.createHttpClient builds an HttpAsyncClient with no version policy, which in HttpClient 5.x negotiates h2-with-upgrade. curl defaults to HTTP/1.1, which is why it succeeds. Probe an explicit h2 upgrade to confirm or refute that a 2.19 node completes it before changing any client code. Diagnostic only. Signed-off-by: Hanyu Wei --- .../workflows/ppl-lint-multiversion-validation.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index d0d18eca2fb..dd89d02cc12 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -374,6 +374,19 @@ jobs: echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" fi done + # The requests above all succeed over HTTP/1.1 (curl's default) yet the + # test client still times out on the same endpoint. The remaining + # difference is protocol negotiation: RestClientBuilder builds an + # HttpAsyncClient with no version policy, which in HttpClient 5.x means + # h2-with-upgrade. Probe an explicit h2 upgrade to see whether this engine + # completes it. + start=$(date +%s) + if curl -sS --http2 --max-time 30 -o /dev/null -w '%{http_version}' \ + "http://localhost:9200/_nodes/plugins" > /tmp/h2.out 2>/tmp/h2.err; then + echo "h2 probe: negotiated HTTP/$(cat /tmp/h2.out) in $(($(date +%s) - start))s" + else + echo "::warning::h2 probe FAILED after $(($(date +%s) - start))s: $(cat /tmp/h2.err)" + fi - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 From 4c80f7b917582d904c9632d24b6cfbb4c82c8407 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:25:39 -0700 Subject: [PATCH 17/39] fix(test): pin the REST test client to HTTP/1.1 Every request from the integ-test REST client to a 2.x engine fails with SocketTimeoutException after the full 60s response timeout, thrown from AbstractSingleCoreIOReactor.execute during client construction -- before a single test runs. The 2.19 leg of the PPL lint multi-version matrix could not observe anything because of it. Root cause: RestClientBuilder.createHttpClient builds its async client with HttpAsyncClientBuilder.create() and never sets a version policy, which in HttpClient 5.x means "negotiate h2". Against a server that supports h2 that is fine; against one that does not, the async I/O reactor stalls rather than falling back to 1.1. Isolated by probing from inside the runner, which ruled out three other explanations in turn: all three of the framework's startup requests (_nodes/plugins, _cat/indices?expand_wildcards=all, _cluster/health) returned HTTP 200 in 0s over curl, so neither the engine, the endpoint, nor the address was at fault. An explicit --http2 probe then split the two legs cleanly: engine 3.5.0 negotiated HTTP/2 leg PASSED engine 2.19.0 negotiated HTTP/1.1 leg timed out at exactly 60s curl falls back cleanly where this client does not. These tests never need h2, so asking for 1.1 up front removes the negotiation entirely and works across every supported engine line. Applied inside each existing config callback rather than as its own setHttpClientConfigCallback call: that setter replaces rather than accumulates, so a separate call would have silently dropped the credentials provider on a secured cluster and the TLS strategy on an https one -- only on the paths where those matter. Signed-off-by: Hanyu Wei --- .../sql/legacy/OpenSearchSQLRestTestCase.java | 44 ++++++++++++++++--- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 267adea43c3..0abe1f4ab7c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; @@ -28,6 +29,7 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; +import org.apache.hc.core5.http2.HttpVersionPolicy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.LogManager; @@ -255,12 +257,39 @@ protected static void configureClient(RestClientBuilder builder, Settings settin credentialsProvider.setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials(userName, password.toCharArray())); - return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return forceHttp11( + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); }); + } else { + builder.setHttpClientConfigCallback(OpenSearchSQLRestTestCase::forceHttp11); } OpenSearchRestTestCase.configureClient(builder, settings); } + /** + * Pin a client to HTTP/1.1. + * + *

{@code RestClientBuilder} builds its async client with no version policy, which in + * HttpClient 5.x means "negotiate h2". Against a server that supports h2 that is fine; against + * one that does not, the async I/O reactor stalls instead of falling back, so every request fails + * with {@code SocketTimeoutException} after the full response timeout — thrown from {@code + * AbstractSingleCoreIOReactor.execute} before a single test runs. + * + *

Live-verified on the PPL lint multi-version matrix: an {@code --http2} probe against engine + * 3.5.0 negotiated HTTP/2 and that leg PASSED, while the same probe against 2.19.0 reported + * HTTP/1.1 and the leg timed out at exactly 60s. curl falls back cleanly; this client does not. + * These tests never need h2, so asking for 1.1 up front removes the negotiation and works across + * every supported engine line. + * + *

Applied INSIDE each config callback rather than as its own {@code + * setHttpClientConfigCallback} call, because that setter replaces rather than accumulates: a + * separate call would silently drop the credentials or TLS configuration set here, and only on + * the paths where it matters. + */ + private static HttpAsyncClientBuilder forceHttp11(HttpAsyncClientBuilder httpClientBuilder) { + return httpClientBuilder.setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_1); + } + protected static void configureHttpsClient( RestClientBuilder builder, Settings settings, HttpHost httpHost) throws IOException { Map headers = ThreadContext.buildDefaultHeaders(settings); @@ -292,12 +321,13 @@ protected static void configureHttpsClient( .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) .build(); - return httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager( - PoolingAsyncClientConnectionManagerBuilder.create() - .setTlsStrategy(tlsStrategy) - .build()); + return forceHttp11( + httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager( + PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build())); } catch (Exception e) { throw new RuntimeException(e); } From 70e7b03d558948618a1de17ab4ab49f8be3378a0 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:29:00 -0700 Subject: [PATCH 18/39] fix(ci): tell a partial engine fix from a full one before advising version scoping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `classifyDrift` runs per query, so a single trigger the engine started accepting looked identical whether the rule's other triggers still failed or not. Its remediation then advised on the whole rule: "set appliesTo.maxVersion just below this version". That advice is wrong whenever the fix was partial — scoping the rule away drops the diagnostics that are still correct, turning a partial engine fix into a shipped false negative on the shapes that remain broken. The default lean was toward version-scoping, so the tool nudged toward the regression. Adds `classifyRelaxationScope`, which decides per RULE per version: every observed trigger relaxed -> engine-relaxed, version-scope-rule some relaxed, others rejected -> engine-partially-relaxed, update-detector The aggregator now buffers per-query findings, collects each trigger's engine verdict, and emits one rule-level verdict that supersedes the per-query ones — so the report shows a single decision rather than one "scope this away" paragraph per trigger. Three details that keep it from producing confident nonsense: - A trigger with no usable verdict counts as neither relaxed nor holding. Counting it as holding would let a timed-out leg read as a partial fix and send someone to narrow a healthy detector; the advice names those triggers and says to re-run. - Only triggers whose contract pinned a rejection can relax. An advisory rule's queries are all valid PPL, so counting them would fabricate a full-fix verdict for a rule the engine never rejected. - The evidence states the tally ("2 of 3 observed trigger(s) relaxed"), and a single-trigger rule gets an explicit warning that a full-fix verdict rests on one observation — the inference the corpus size cannot yet support. Signed-off-by: Hanyu Wei --- .../__tests__/aggregate-versions.test.mjs | 127 ++++++++++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 94 ++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 70 ++++++++- scripts/ppl-lint/drift.mjs | 141 ++++++++++++++++++ 4 files changed, 430 insertions(+), 2 deletions(-) diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 65606dbb926..8100829eb9d 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -215,6 +215,133 @@ test('a version where only one engine relaxed is red, and names just that versio assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); }); +// --- partial vs full relaxation, end to end --------------------------------- +// +// Driven through the real script because the bug this guards is in the AGGREGATION: +// `classifyDrift` is per-query and cannot see the other triggers, so the rollup has +// to happen here or the advice is wrong whenever a rule has more than one trigger. + +/** A two-trigger contract: the shape that makes partial-vs-full decidable. */ +function writeTwoTriggerContracts() { + const dir = makeTmp('ppl-lint-contracts-multi-'); + const spec = { + ...SPEC, + queries: { + triggerA: { role: 'trigger', query: 'union [ source={{index}} ]' }, + triggerB: { role: 'trigger', query: 'union [ source={{index}} ] extra' }, + control: { role: 'control', query: 'union [ source={{index}} ] [ source={{index}} ]' }, + }, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + triggerA: { + detectorCount: 1, + severity: 'error', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400, error: REJECTION } }, + }, + triggerB: { + detectorCount: 1, + severity: 'error', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400, error: REJECTION } }, + }, + control: { detectorCount: 0, backend: { kind: 'result-shape', httpStatus: 200 } }, + }, + }, + ], + }; + fs.writeFileSync(path.join(dir, 'union.spec.json'), JSON.stringify(spec)); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ + schemaVersion: 3, + contracts: ['union.spec.json'], + defaultError: ['union.spec.json'], + }) + ); + return dir; +} + +test('ALL triggers relaxing is a full fix: one rule-level version-scope finding', () => { + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { status, report } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal(status, 1); + // Exactly ONE finding, not one per trigger: the decision is per rule. + assert.equal(report.drifts.length, 1); + const drift = report.drifts[0]; + assert.equal(drift.driftClass, 'engine-relaxed'); + assert.equal(drift.remediation.action, 'version-scope-rule'); + assert.deepEqual(drift.scope.relaxed.sort(), ['triggerA', 'triggerB']); + assert.deepEqual(drift.scope.holding, []); +}); + +test('SOME triggers relaxing is a partial fix: narrow the detector, do NOT scope', () => { + // The regression this pins: acting on the per-query view would advise + // maxVersion < 3.7, dropping the diagnostic for triggerB which the engine STILL + // rejects — converting a partial engine fix into a shipped false negative. + const legs = { + '3.7.0': writeLeg({ + version: '3.7.0', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { status, report, stdout } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal(status, 1); + const partial = report.drifts.find((d) => d.driftClass === 'engine-partially-relaxed'); + assert.ok(partial, 'a partial relaxation must be classified as such'); + assert.equal(partial.remediation.action, 'update-detector'); + assert.deepEqual(partial.scope.relaxed, ['triggerA']); + assert.deepEqual(partial.scope.holding, ['triggerB']); + // No finding may survive that tells the engineer to version-scope this rule. + assert.equal( + report.drifts.filter((d) => d.remediation.action === 'version-scope-rule').length, + 0, + 'a partial fix must never advise version-scoping' + ); + assert.match(stdout, /Do NOT scope/); +}); + +test('an unobserved trigger does not fake a partial fix', () => { + // If the unobserved trigger were counted as "still rejects", this would classify + // as partial and send someone to narrow a detector on the strength of a leg that + // never answered. + const legs = { + '3.7.0': writeLegWithTransportError({ + version: '3.7.0', + erroredQuery: 'triggerB', + cases: { + triggerA: { detector: 1, rejected: false }, + triggerB: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }), + }; + const { report } = run({ contracts: writeTwoTriggerContracts(), legs }); + assert.equal( + report.drifts.filter((d) => d.driftClass === 'engine-partially-relaxed').length, + 0, + 'an unobserved trigger must not be counted as holding' + ); + const relaxed = report.drifts.find((d) => d.driftClass === 'engine-relaxed'); + assert.ok(relaxed); + assert.deepEqual(relaxed.scope.unobserved, ['triggerB']); + assert.match(relaxed.remediation.detail, /produced no verdict/); +}); + test('a rule out of scope on an older engine that accepts is not drift', () => { const legs = healthyLegs(); // 3.6 predates the rule's minVersion and accepts the query: intended silence. diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 4c8d4e6f4dc..251ecb3c9bf 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -23,6 +23,7 @@ import { DRIFT_CLASSES, REMEDIATIONS, classifyDrift, + classifyRelaxationScope, formatDriftReport, parseVersion, suggestParserRules, @@ -436,6 +437,99 @@ test('rename suggestions prefer containment then near spellings', () => { assert.deepEqual(suggestParserRules('rexCommand', ['whereClause', 'sortCommand'], 3), []); }); +// --- partial vs full relaxation ---------------------------------------------- +// +// The distinction these tests protect: a rule whose triggers ALL relaxed should be +// scoped away from the version; a rule where only SOME relaxed must NOT be, because +// scoping it would drop the diagnostics that are still correct. Getting this +// backwards converts a partial engine fix into a shipped false negative, so each +// branch is pinned including the advice text that names the wrong action. + +const scopeBase = { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + detectorFlagged: true, +}; + +test('no relaxed trigger yields no rule-level finding', () => { + assert.equal( + classifyRelaxationScope({ ...scopeBase, relaxedTriggers: [], holdingTriggers: ['a', 'b'] }), + null + ); +}); + +test('every trigger relaxed is a FULL fix and advises version scoping', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['hyphen', 'leading-digit'], + holdingTriggers: [], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.VERSION_SCOPE_RULE); + assert.match(drift.evidence, /FULL fix, 2 of 2 observed trigger\(s\) relaxed/); +}); + +test('some triggers still rejected is a PARTIAL fix and advises the detector, NOT scoping', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['hyphen'], + holdingTriggers: ['leading-digit', 'all-digits'], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /PARTIAL fix, 1 of 3 observed trigger\(s\) relaxed/); + // The advice must say the wrong action out loud. An engineer reading only the + // action verb could still reach for maxVersion, which is the regression. + assert.match(drift.remediation.detail, /Do NOT scope .* away from 3\.8\.0/); + assert.match(drift.remediation.detail, /false NEGATIVE/); +}); + +test('a single-trigger rule warns that a FULL verdict rests on one observation', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['only-one'], + holdingTriggers: [], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.match(drift.remediation.detail, /only 1 trigger/); + assert.match(drift.remediation.detail, /confirm with more shapes/); +}); + +test('unobserved triggers are excluded from the tally and named in the advice', () => { + // The trap: counting an unobserved trigger as "holding" turns a dead leg into a + // partial fix and sends someone to narrow a healthy detector. + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['a'], + holdingTriggers: [], + unobservedTriggers: ['b'], + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED, 'must not read as partial'); + assert.match(drift.evidence, /1 of 1 observed trigger\(s\) relaxed/); + assert.match(drift.evidence, /1 trigger\(s\) produced no verdict \(b\) and were NOT counted/); + assert.match(drift.remediation.detail, /re-run it before acting/); +}); + +test('a silent detector on a fully relaxed rule needs no linter change', () => { + const drift = classifyRelaxationScope({ + ...scopeBase, + relaxedTriggers: ['a', 'b'], + holdingTriggers: [], + detectorFlagged: false, + }); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); +}); + +test('the per-query relaxed finding is marked supersedable', () => { + // The aggregator drops these in favour of the rule-level verdict; without the + // marker it would report both, and the per-query one gives the wrong action. + const drift = classifyDrift( + agreeingTrigger({ observed: { detectorCount: 1, severities: ['error'], backendRejected: false } }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.equal(drift.supersededBy, DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED); +}); + // --- report ------------------------------------------------------------------ test('the report groups by action, most urgent first', () => { diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index a0ab0f69613..d7d6b25731f 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -37,6 +37,8 @@ import { emitAnnotations } from './annotate.mjs'; import { classifyDrift, classifyGrammarDrift, + classifyRelaxationScope, + DRIFT_CLASSES, formatDriftReport, versionInAppliesTo, } from './drift.mjs'; @@ -504,6 +506,19 @@ function main() { // no action, unusable means something did not answer and needs a re-run. let ruleNotApplicable = 0; const unusable = []; + // Per-trigger engine verdicts for this rule on this leg, so a relaxation can + // be judged across the WHOLE rule rather than one query at a time. A single + // relaxed trigger cannot distinguish a full engine fix (scope the rule away) + // from a partial one (narrow the detector), and those actions are opposites — + // acting on the per-query view ships a false negative in the partial case. + // `unobserved` is kept apart from `holding` on purpose: a trigger that never + // answered must not be counted as "still rejects", or a timed-out leg would + // read as a partial fix and send someone to narrow a healthy detector. + const relaxedTriggers = []; + const holdingTriggers = []; + const unobservedTriggers = []; + let relaxedDetectorFlagged = false; + const perQueryDrifts = []; for (const [queryName, expected] of Object.entries(expectation.queries || {})) { const queryDef = (spec.queries || {})[queryName]; if (!queryDef) { @@ -550,11 +565,28 @@ function main() { unusable.push( `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` ); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } continue; } compared++; if (role === 'trigger') { triggersCompared++; + // Bucket this trigger by what the ENGINE did, but only where the contract + // pinned a rejection — a trigger pinned as accepted (an advisory rule like + // head-without-sort, whose queries are all valid PPL) never "relaxes", and + // counting it as relaxed would fabricate a full-fix verdict for a rule the + // engine was never rejecting in the first place. + const pinnedRejection = (expected.backend && expected.backend.kind) === 'rejection'; + if (pinnedRejection) { + if (observed.backendRejected === false) { + relaxedTriggers.push(queryName); + if ((observed.detectorCount || 0) > 0) relaxedDetectorFlagged = true; + } else if (observed.backendRejected === true) { + holdingTriggers.push(queryName); + } + } } const drift = classifyDrift({ @@ -581,16 +613,50 @@ function main() { // string identifies WHICH `expectations[]` entry produced this finding, // so a `update-contract` annotation can land on that entry's line rather // than at the top of the file. - drifts.push({ + // Buffered rather than pushed: a relaxation finding is only final once + // every trigger has been seen, because the rule-level rollup below + // replaces the per-query ones with a single full-vs-partial verdict. + perQueryDrifts.push({ ...drift, enforced: isEnforced, contractFile: file, expectationRange: expectation.version, expectationEngine: expectation.engine, }); - ruleDrifts++; } } + + // Every trigger has now been observed, so a relaxation can be judged for the + // rule as a whole. This supersedes the per-query `engine-relaxed` findings — + // they each said "scope this rule away from this version", which is the wrong + // action whenever another trigger still rejects. + const relaxationScope = classifyRelaxationScope({ + ruleId, + version: leg.version, + relaxedTriggers, + holdingTriggers, + unobservedTriggers, + detectorFlagged: relaxedDetectorFlagged, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + }); + const kept = relaxationScope + ? perQueryDrifts.filter((d) => d.supersededBy !== DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED) + : perQueryDrifts; + for (const drift of kept) { + drifts.push(drift); + ruleDrifts++; + } + if (relaxationScope) { + drifts.push({ + ...relaxationScope, + enforced: isEnforced, + contractFile: file, + expectationRange: expectation.version, + expectationEngine: expectation.engine, + }); + ruleDrifts++; + } // "agree" has to mean "we compared the rule's claim and it held". A rule // whose every case lost its engine verdict (a timed-out leg) or its detector // row (a runner that died mid-corpus) has proven nothing — and so has one diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index 43c9979bb6d..3fe9c6fa5dd 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -32,6 +32,7 @@ export const DRIFT_CLASSES = { GRAMMAR_RULE_MISSING: 'grammar-rule-missing', ENGINE_RELAXED: 'engine-relaxed', + ENGINE_PARTIALLY_RELAXED: 'engine-partially-relaxed', ENGINE_TIGHTENED: 'engine-tightened', ENGINE_MESSAGE_CHANGED: 'engine-message-changed', DETECTOR_SILENT: 'detector-silent', @@ -241,6 +242,139 @@ export function classifyGrammarDrift({ }; } +/** + * Decide, for ONE rule on ONE engine version, whether an observed relaxation is + * total or partial — the difference between "scope the rule away from this + * version" and "narrow the detector". + * + * `classifyDrift` sees a single query, so it cannot tell these apart: one trigger + * that the engine now accepts looks identical whether the rule's other triggers + * still fail or not. Acting on that one query is actively harmful in the partial + * case, because scoping the rule out of the version drops the diagnostics that + * are STILL correct — turning a partial engine fix into a false negative on the + * shapes that remain broken. That is why this runs over the whole rule. + * + * Inputs are the per-trigger verdicts the caller already gathered: + * relaxed engine ACCEPTS a trigger the contract pinned as rejected + * holding engine still REJECTS the trigger + * Triggers with no usable verdict are passed as neither, and are reported as the + * reason a verdict is being withheld rather than silently treated as `holding` + * (which would read a dead leg as a partial fix and narrow a healthy detector). + * + * Returns null when nothing relaxed — the caller's per-query drifts stand on + * their own. Otherwise returns ONE rule-level drift that supersedes the + * per-query `engine-relaxed` findings, so the report shows a single decision + * instead of one "scope this away" paragraph per trigger. + * + * @param {object} input + * @param {string[]} input.relaxedTriggers trigger names the engine now accepts + * @param {string[]} input.holdingTriggers trigger names the engine still rejects + * @param {string[]} [input.unobservedTriggers] triggers with no comparable verdict + * @param {boolean} [input.detectorFlagged] did the detector fire on any relaxed trigger + */ +export function classifyRelaxationScope({ + ruleId, + version, + relaxedTriggers = [], + holdingTriggers = [], + unobservedTriggers = [], + detectorFlagged = false, + wiring, + detectorPath, +}) { + if (relaxedTriggers.length === 0) { + return null; + } + + const where = `${ruleId} @ ${version}`; + const base = { + ruleId, + version, + driftVersion: version, + role: 'trigger', + scope: { + relaxed: [...relaxedTriggers], + holding: [...holdingTriggers], + unobserved: [...unobservedTriggers], + }, + }; + // How thin is the basis for a "fully relaxed" claim? A rule with ONE pinned + // trigger that relaxes proves only that one shape changed; calling that "the + // behavior is gone" is a much bigger inference than the data supports. The + // count goes in the evidence either way so the reader can judge it, rather + // than the tool quietly presenting 1-of-1 as though it were 5-of-5. + const observed = relaxedTriggers.length + holdingTriggers.length; + const basis = `${relaxedTriggers.length} of ${observed} observed trigger(s) relaxed`; + + // --- Partial: some triggers relaxed, others still rejected ------------------ + // The engine fixed part of the condition. Scoping the rule out of this version + // would ship a false negative on everything in `holding`, so the action is to + // narrow the detector to the shapes that still fail. + if (holdingTriggers.length > 0) { + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS ${relaxedTriggers.length} of this rule's triggers ` + + `(${relaxedTriggers.join(', ')}) but still REJECTS ${holdingTriggers.length} ` + + `(${holdingTriggers.join(', ')}) — a PARTIAL fix, ${basis}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Do NOT scope "${ruleId}" away from ${version}: the engine still rejects ` + + `${holdingTriggers.join(', ')}, so a maxVersion below ${version} would drop diagnostics that ` + + `are still correct and ship a false NEGATIVE there. Narrow the detector so it stops matching ` + + `the now-valid shape(s) (${relaxedTriggers.join(', ')}) while still flagging the rest, then ` + + `re-pin the ${version} expectation for the relaxed trigger(s) to detectorCount 0.`, + }, + }; + } + + // --- Full: every observed trigger relaxed ----------------------------------- + // Nothing the rule claims is still true on this engine. Version-scoping is now + // the right action — with the caveat that "every observed trigger" is only as + // strong as the trigger count, which the evidence states. + return { + ...base, + driftClass: DRIFT_CLASSES.ENGINE_RELAXED, + evidence: + `${where}: engine ${version} now ACCEPTS every observed trigger for this rule ` + + `(${relaxedTriggers.join(', ')}) — a FULL fix, ${basis}` + + (unobservedTriggers.length > 0 + ? `; ${unobservedTriggers.length} trigger(s) produced no verdict (${unobservedTriggers.join(', ')}) ` + + `and were NOT counted` + : '') + + '.', + remediation: detectorFlagged + ? { + action: REMEDIATIONS.VERSION_SCOPE_RULE, + target: OSD_PATHS.catalog, + detail: + `Every trigger this contract pins is now valid on ${version}, so "${ruleId}" is a FALSE ` + + `POSITIVE there. Set appliesTo.maxVersion just below ${version} to keep protecting users on ` + + `older engines; if no supported engine rejects any trigger any more, set "enabled": false and ` + + `drop the detector. Then re-pin the ${version} expectations to detectorCount 0.` + + (observed < 2 + ? ` NOTE: this rule pins only ${observed} trigger, so "fully relaxed" rests on a single ` + + `observation — confirm with more shapes of the same condition before scoping the rule away.` + : '') + + (unobservedTriggers.length > 0 + ? ` NOTE: ${unobservedTriggers.length} trigger(s) produced no verdict on this leg; re-run it ` + + `before acting, since one of them may still reject.` + : ''), + } + : { + action: REMEDIATIONS.UPDATE_CONTRACT, + target: 'this contract file', + detail: + `The detector already stays silent on ${version}, so no linter change is needed. Re-pin the ` + + `${version} expectations to detectorCount 0 / backend.kind "result-shape" to record the ` + + `engine's new behavior.`, + }, + }; +} + /** * Classify one query's outcome on one engine version. * @@ -379,6 +513,13 @@ export function classifyDrift(input) { if (role === 'trigger' && expectRejection && backendRejected === false) { return { ...base, + // A single relaxed trigger cannot tell a full fix from a partial one, and the + // two need OPPOSITE actions (scope the rule away vs narrow the detector). The + // caller aggregates every trigger through `classifyRelaxationScope` and drops + // the findings carrying this marker in favour of that one rule-level verdict. + // Kept as a finding rather than returning null so a caller that does not + // aggregate still reports the relaxation instead of silently passing. + supersededBy: DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED, driftClass: DRIFT_CLASSES.ENGINE_RELAXED, evidence: `${where}: engine ${version} now ACCEPTS a query the contract pinned as rejected ` + From aef2c1a5e0b2dc96aebfc77283ea5f773f171d68 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:29:18 -0700 Subject: [PATCH 19/39] feat(ci): harvest a discovery corpus so trigger coverage can support a scope decision MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The partial-vs-full relaxation verdict added in the previous commit needs several triggers per rule to be meaningful. The enforced corpus has 27 queries across 11 rules — mostly one trigger each — so for most rules "every trigger relaxed" is a single observation, and the report has to say so rather than present it as proof. OSD's own lint tests already contain the variety: whoever wrote each detector wrote several queries that should fire and several that should not, grouped by rule. `invalid-capture-group-name` has three distinct reasons a name is invalid there versus one in the contract. Harvesting them yields 109 queries across 12 rules — 4x the enforced corpus — at no authoring cost. harvest-queries.mjs extract PPL literals from OSD's lint tests; attribute each to the rule whose describe(...) block encloses it, unescape JS string escapes, remap the index onto the fixture probe-discovery-backend.mjs POST each query to /_plugins/_ppl and record the verdict (no Gradle, no test cluster — there is nothing to assert) label-discovery.mjs derive each role from real detector output and report detector/engine disagreements Roles are derived; expectations never are. An auto-derived expectation could only confirm current behavior, locking in whatever the detector does today including its bugs. `--specs-out` writes the corpus as ordinary spec files so the EXISTING detector runner produces real counts unmodified — the enforced check's behavior is untouched. Promotion into the enforced corpus stays a human writing a spec entry. The new `discovery` job is `continue-on-error` and the labeler always exits zero: a finding here is a lead, not a proven defect, and failing unrelated PRs on an auto-generated guess would destroy the check's credibility. Verified end to end against a live 3.8 cluster. That run found a defect in the labeler itself: `head-without-sort` and `rex-scan-cost` are `info` rules flagging non-determinism and scan cost, which the engine executes happily and will never reject — so "accepted + flagged" is the rule working as designed, not a false positive. Severity is the discriminator, read from what the detector actually emitted. With that fixed the run reports 0 findings on 109 queries, 75 rejections suppressed as uninformative (unknown field, missing index, unsupported command, syntax error) and 2 advisory triggers excluded. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 210 +++++++ scripts/ppl-lint/README.md | 119 +++- .../__tests__/harvest-queries.test.mjs | 240 ++++++++ .../__tests__/label-discovery.test.mjs | 242 ++++++++ .../probe-discovery-backend.test.mjs | 66 ++ scripts/ppl-lint/harvest-queries.mjs | 576 ++++++++++++++++++ scripts/ppl-lint/label-discovery.mjs | 467 ++++++++++++++ scripts/ppl-lint/probe-discovery-backend.mjs | 196 ++++++ 8 files changed, 2111 insertions(+), 5 deletions(-) create mode 100644 scripts/ppl-lint/__tests__/harvest-queries.test.mjs create mode 100644 scripts/ppl-lint/__tests__/label-discovery.test.mjs create mode 100644 scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs create mode 100644 scripts/ppl-lint/harvest-queries.mjs create mode 100644 scripts/ppl-lint/label-discovery.mjs create mode 100644 scripts/ppl-lint/probe-discovery-backend.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index dd89d02cc12..2438e665fbe 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -115,6 +115,7 @@ jobs: outputs: released: ${{ steps.plan.outputs.released }} compiled: ${{ steps.plan.outputs.compiled }} + discovery_engine: ${{ steps.plan.outputs.discovery_engine }} osd_repo: ${{ steps.plan.outputs.osd_repo }} osd_ref: ${{ steps.plan.outputs.osd_ref }} steps: @@ -165,6 +166,24 @@ jobs: " echo "compiled=$compiled" >> "$GITHUB_OUTPUT" echo "Compiled-surface legs: $compiled" >> "$GITHUB_STEP_SUMMARY" + + # Discovery runs against ONE engine — the newest released version in the + # matrix. It is a lead-generator, not a version-drift check, so paying for + # a full matrix would multiply cost without adding signal: a false positive + # found on the newest engine is the one users hit soonest, and per-version + # differences are already the enforced corpus's job. + discovery_engine=$(echo "$released" | python3 -c " + import json,sys + v=json.load(sys.stdin) + # Newest by semver, not list order, so a reordered matrix cannot silently + # point discovery at an old engine. + def key(s): + parts=[int(p) for p in s.split('-')[0].split('.') if p.isdigit()] + return parts + [0]*(3-len(parts)) + print(sorted(v,key=key)[-1]) + ") + echo "discovery_engine=$discovery_engine" >> "$GITHUB_OUTPUT" + echo "Discovery engine: \`$discovery_engine\`" >> "$GITHUB_STEP_SUMMARY" # Same precedence as the sibling workflow: dispatch input, then repo # variable, then the canonical upstream default. echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" @@ -674,3 +693,194 @@ jobs: legs/**/detector-report.json legs/**/detector.log legs/**/target.json + + # Discovery: harvest queries from OSD's own lint tests, run both halves over them, + # and report detector/engine disagreements as LEADS. + # + # Why this is separate from `detect`, and why it can never fail the build: + # + # The enforced corpus is hand-pinned — every expectation is a reviewed claim, which + # is what lets a mismatch red the build. That corpus is also small (about one + # trigger per rule), and `classifyRelaxationScope` needs SEVERAL triggers per rule + # to tell a FULL engine fix (version-scope the rule away) from a PARTIAL one + # (narrow the detector). Those need opposite actions, so with one trigger the + # advice can be confidently wrong. + # + # This job supplies that trigger variety from queries OSD's own detector authors + # already wrote. It pins NOTHING: roles are derived from real detector output and + # the engine supplies the other half, so no expectation is ever auto-generated. + # An auto-derived expectation could only confirm current behavior — locking in + # whatever the detector does today, bugs included. + # + # `continue-on-error` AND a zero exit from the labeler: a finding here is a lead to + # investigate, not a proven defect, and blocking unrelated PRs on an auto-generated + # guess would poison the whole check's credibility. + discovery: + name: Discovery corpus (harvested, not enforced) + # Only `plan`, for the OSD target and the engine version. Deliberately NOT the + # observe legs: discovery runs its own engine and harvests its own queries, so + # depending on them would idle this job behind ~30 minutes of matrix work it + # never reads, and a failed leg would block a report that does not need it. + needs: plan + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 40 + services: + opensearch: + image: opensearchproject/opensearch:${{ needs.plan.outputs.discovery_engine }} + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ needs.plan.outputs.osd_repo }} + ref: ${{ needs.plan.outputs.osd_ref }} + path: .ci/OpenSearch-Dashboards + + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + # The rule id list comes from the OSD catalog being validated, not a hardcoded + # copy: attribution keys off `describe('')` titles, so a stale list + # would silently stop harvesting queries for any newly added rule. + - name: Harvest the discovery corpus from OSD's lint tests + run: | + set -euo pipefail + node -e " + const c = require('./.ci/OpenSearch-Dashboards/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); + process.stdout.write(JSON.stringify(c.map((r) => r.id))); + " > /tmp/catalog-rules.json + node scripts/ppl-lint/harvest-queries.mjs \ + --osd .ci/OpenSearch-Dashboards \ + --catalog-rules @/tmp/catalog-rules.json \ + --index opensearch-sql_test_index_account \ + --out "$GITHUB_WORKSPACE/discovery-corpus.json" \ + --specs-out "$GITHUB_WORKSPACE/discovery-specs" + + # Seed the one index every harvested query was rewritten onto. Without it the + # engine rejects everything with IndexNotFoundException — which the labeler + # would correctly suppress as uninformative, yielding a run that reports + # nothing at all. + - name: Seed the fixture index + run: | + set -euo pipefail + for i in $(seq 1 40); do + curl -sf http://localhost:9200 > /dev/null && break + echo "waiting for engine (${i}/40)..." + sleep 5 + done + curl -sf -X PUT "http://localhost:9200/opensearch-sql_test_index_account" \ + -H 'content-type: application/json' -d '{ + "mappings": { "properties": { + "account_number": { "type": "long" }, + "balance": { "type": "long" }, + "age": { "type": "integer" }, + "status": { "type": "keyword" }, + "firstname": { "type": "text" }, + "lastname": { "type": "text" }, + "msg": { "type": "text" }, + "body": { "type": "text" }, + "raw": { "type": "object", "enabled": false } + } } + }' + curl -sf -X POST "http://localhost:9200/opensearch-sql_test_index_account/_doc?refresh=true" \ + -H 'content-type: application/json' \ + -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' + + - name: Run the detectors over the discovery corpus + working-directory: .ci/OpenSearch-Dashboards + run: | + set -uo pipefail + # A non-zero exit is EXPECTED and ignored: the generated specs carry + # placeholder expectations, so the runner reports a "failure" for every + # query whose real diagnostic count differs. Only the report is read. + PPL_LINT_SURFACE=compiled-simplified \ + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + > "$GITHUB_WORKSPACE/discovery-detector.log" 2>&1 || true + if [ ! -f "$GITHUB_WORKSPACE/discovery-detector-report.json" ]; then + echo "::warning::the detector runner produced no discovery report; skipping." + tail -50 "$GITHUB_WORKSPACE/discovery-detector.log" || true + fi + + - name: Probe the engine with the discovery corpus + run: | + set -euo pipefail + node scripts/ppl-lint/probe-discovery-backend.mjs \ + --corpus discovery-corpus.json \ + --endpoint http://localhost:9200 \ + --out discovery-backend-report.json + + - name: Label and report + run: | + set -euo pipefail + if [ ! -f discovery-detector-report.json ]; then + echo "::warning::no detector report; nothing to label." + exit 0 + fi + node scripts/ppl-lint/label-discovery.mjs \ + --corpus discovery-corpus.json \ + --detector discovery-detector-report.json \ + --backend discovery-backend-report.json \ + --version "${{ needs.plan.outputs.discovery_engine }}" \ + --out discovery-findings.json \ + --summary "$GITHUB_STEP_SUMMARY" + + - name: Upload discovery artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-discovery + path: | + discovery-corpus.json + discovery-findings.json + discovery-detector-report.json + discovery-backend-report.json + discovery-detector.log diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index f20633768d4..a7cbd31714b 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -292,11 +292,36 @@ Every finding names a drift class, the evidence, and one remediation action: Drift classes: `grammar-rule-missing` (a parser rule the detector walks was renamed or removed — the finding names the closest current rule names), -`engine-relaxed` / `engine-tightened` (the engine's verdict flipped), -`engine-message-changed` (same verdict, reworded error), `detector-silent` / -`detector-noisy` (false negative / false positive), `version-scope-too-narrow` -(the engine rejects but the rule is scoped away from that version, so users see no -diagnostic), and `severity-mismatch`. +`engine-relaxed` / `engine-partially-relaxed` / `engine-tightened` (the engine's +verdict flipped), `engine-message-changed` (same verdict, reworded error), +`detector-silent` / `detector-noisy` (false negative / false positive), +`version-scope-too-narrow` (the engine rejects but the rule is scoped away from +that version, so users see no diagnostic), and `severity-mismatch`. + +#### Full vs partial relaxation: scope the rule, or narrow the detector? + +When an engine starts accepting a query a rule flags, the fix depends on a question +a single query cannot answer: is the behavior **fully** gone on that version, or +only **partially**? + +- **Every trigger relaxed** → `engine-relaxed`, action `version-scope-rule`. Nothing + the rule claims is still true on that engine, so bound it with `maxVersion`. +- **Some triggers relaxed, others still rejected** → `engine-partially-relaxed`, + action `update-detector`. The engine fixed *part* of the condition. Scoping the + rule away here would drop the diagnostics that are still correct, converting a + partial engine fix into a shipped **false negative**. Narrow the detector so it + stops matching the now-valid shapes while still flagging the rest. + +This is decided per rule, not per query: the aggregator collects every trigger's +engine verdict for a rule on a leg, then emits **one** rule-level finding that +supersedes the per-query ones. A trigger with no verdict is counted as neither — +treating it as "still rejects" would let a timed-out leg masquerade as a partial fix +and send someone to narrow a healthy detector. + +The evidence always states the tally (`2 of 3 observed trigger(s) relaxed`), and a +rule with only one pinned trigger gets an explicit warning that a "fully relaxed" +verdict rests on a single observation. That is the gap the discovery corpus below +closes. Four guards keep the check from passing vacuously. Each exists because "we could not check" must never render as "it is fine": @@ -385,6 +410,90 @@ anywhere: node --test "scripts/ppl-lint/__tests__/*.test.mjs" ``` +## Discovery corpus (harvested, never enforced) + +The enforced corpus is hand-pinned, which is what lets a mismatch red the build — +and also why it is small (about one trigger per rule). One trigger is not enough to +tell a full engine fix from a partial one, so the `discovery` job builds a second, +much larger corpus that pins nothing. + +``` +harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-frontend-contract.mjs ──▶ detector report + + discovery-specs/ └──▶ probe-discovery-backend.mjs ─▶ backend report + │ + label-discovery.mjs ──▶ findings + trigger coverage +``` + +1. **Harvest.** `harvest-queries.mjs` extracts PPL literals from OSD's own lint test + suite and attributes each to the rule whose `describe(...)` block encloses it + (matched as a prefix, so `describe('rex-scan-cost (compiled surface)')` counts). + A query with no rule-owning ancestor is recorded unattributed and dropped rather + than guessed at. Indices are rewritten onto the fixture index; JS string escapes + are unescaped so the query matches what the test actually linted. Against OSD + `main` today this yields **~109 queries across 12 rules** versus 27 across 11 in + the enforced corpus. +2. **Observe both halves.** `--specs-out` writes the corpus as ordinary spec files so + the **existing** detector runner produces real diagnostic counts with no changes to + it; a non-zero exit is expected there and ignored, because the generated + expectations are placeholders. `probe-discovery-backend.mjs` sends each query to + `POST /_plugins/_ppl` directly — no Gradle, no test cluster, since there is + nothing to assert. +3. **Label and report.** `label-discovery.mjs` derives each role from real detector + output (fired → trigger, silent → control) and reports disagreements. + +Roles are derived; **expectations never are**. An auto-derived expectation could only +confirm current behavior, locking in whatever the detector does today including its +bugs. Promotion into the enforced corpus stays a human writing a spec entry. + +### What it reports, and how much to trust it + +| Detector | Engine | Reported as | +| --- | --- | --- | +| fires (error/warning) | accepts | `possible-false-positive` — nearly conclusive | +| fires (**info** only) | accepts | nothing — advisory rules are never contradicted by acceptance | +| silent | rejects | `possible-false-negative` — weak, verify first | +| either | no verdict | nothing; the query is labelled but claims nothing | + +The asymmetry is deliberate. A query the engine *ran* successfully but the linter +called broken is unambiguous. A rejection may be for a reason unrelated to the rule, +so rejections matching an unknown field, a missing index, an unsupported command, or +a syntax error are **suppressed** rather than reported — without that filter every +harvested query naming an invented field becomes a finding and buries the real ones. +Suppression never applies to the false-positive side. + +Advisory (`info`) rules are the other exclusion, and it was found by running this +against a live 3.8 engine: `head-without-sort` and `rex-scan-cost` flag +non-determinism and scan cost, which the engine executes happily and will never +reject. For those, "accepted + flagged" is the rule working as designed. Severity is +the discriminator because it already encodes the claim — only an error/warning rule +asserts the engine will refuse the query, and only such a claim can be contradicted +by acceptance. The judgement uses the severities the detector actually emitted, so a +mixed-severity diagnostic is still reported. + +The report also prints per-rule trigger counts and whether each rule has enough +(≥2) to support a scope decision. That table is the direct input to the full-vs-partial +question above: a rule showing **1 trigger** cannot distinguish the two, and a rule +showing **0** was not observed at all. + +This job is `continue-on-error: true` and the labeler always exits zero. A finding +here is a lead, not a proven defect; failing unrelated PRs on an auto-generated +guess would destroy the check's credibility. It runs against one engine (the newest +in the matrix) because it generates leads rather than checking version drift. + +```bash +# Locally, against a running cluster and an OSD checkout: +node -e "const c=require('/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); + process.stdout.write(JSON.stringify(c.map(r=>r.id)))" > /tmp/rules.json +node scripts/ppl-lint/harvest-queries.mjs --osd --catalog-rules @/tmp/rules.json \ + --index opensearch-sql_test_index_account --out /tmp/corpus.json --specs-out /tmp/specs +( cd && PPL_LINT_SURFACE=compiled-simplified PPL_LINT_CONTRACT_DIR=/tmp/specs \ + PPL_LINT_SCHEDULE=nightly PPL_LINT_REPORT=/tmp/detector.json \ + node -r ./src/setup_node_env "$PWD/../sql/scripts/ppl-lint/run-frontend-contract.mjs" || true ) +node scripts/ppl-lint/probe-discovery-backend.mjs --corpus /tmp/corpus.json --out /tmp/backend.json +node scripts/ppl-lint/label-discovery.mjs --corpus /tmp/corpus.json \ + --detector /tmp/detector.json --backend /tmp/backend.json --out /tmp/findings.json +``` + ## Interpreting a failure | Failure | Meaning | diff --git a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs new file mode 100644 index 00000000000..fc9bf7d8276 --- /dev/null +++ b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs @@ -0,0 +1,240 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the discovery-corpus harvester. + * + * node --test scripts/ppl-lint/__tests__/harvest-queries.test.mjs + * + * The harvester's job is to attribute a query to the rule that owns it and to hand + * the labeler something the cluster can actually run. Both have a wrong-answer mode + * that is worse than dropping the query: a misattributed query produces a + * "disagreement" for a rule that never claimed anything about it, and an + * unremapped index produces a rejection that reads as engine behavior. Those two + * failure modes are what these tests pin. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + harvestFile, + referencedIdentifiers, + remapIndex, + ruleFromDescribeTitle, + toRunnerSpecs, +} from '../harvest-queries.mjs'; + +const RULES = [ + 'invalid-capture-group-name', + 'field-validation', + 'rex-scan-cost', + 'head-without-sort', + 'division-by-zero', +]; + +// --- attribution ------------------------------------------------------------- + +test('an exact describe title names its rule', () => { + assert.equal(ruleFromDescribeTitle('rex-scan-cost', RULES), 'rex-scan-cost'); +}); + +test('a suffixed title still names its rule', () => { + // OSD's real titles: `describe('rex-scan-cost (compiled surface)')`. Requiring an + // exact match dropped ~75% of harvestable queries. + assert.equal(ruleFromDescribeTitle('rex-scan-cost (compiled surface)', RULES), 'rex-scan-cost'); + assert.equal( + ruleFromDescribeTitle('field-validation alternate-source suppression', RULES), + 'field-validation' + ); +}); + +test('a title that merely mentions a rule mid-sentence does NOT claim it', () => { + // Prefix-only matching is the guard: this describe sits under some OTHER rule and + // must not steal attribution, or its queries get judged against the wrong rule. + assert.equal(ruleFromDescribeTitle('does not fire on rex-scan-cost candidates', RULES), null); +}); + +test('a longer rule id wins over a prefix of itself', () => { + const rules = ['field-validation', 'field-validation-shape']; + assert.equal(ruleFromDescribeTitle('field-validation-shape cases', rules), 'field-validation-shape'); +}); + +test('a rule id must end at a word boundary', () => { + assert.equal(ruleFromDescribeTitle('head-without-sorting quirks', RULES), null); +}); + +test('the innermost rule-owning describe wins', () => { + const source = ` + describe('PPL silent-failure lint rules (compiled surface)', () => { + describe('division-by-zero', () => { + it('flags it', () => { + expect(ids('source=logs | eval x = a / 0')).toContain('division-by-zero'); + }); + }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].ruleId, 'division-by-zero'); +}); + +test('a query with no rule-owning ancestor is recorded unattributed, not guessed', () => { + const source = ` + describe('some unrelated suite', () => { + it('x', () => { lint('source=logs | head 10'); }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].ruleId, null); +}); + +// --- extraction -------------------------------------------------------------- + +test('JS string escapes are unescaped to the runtime query', () => { + // A test source containing '(?\\\\d+)' is the 4 chars `\\d+` at runtime, which + // is what the detector and the engine both see. Leaving the JS layer escaped + // sends a different query than the test actually linted. + const source = String.raw` + describe('invalid-capture-group-name', () => { + it('x', () => { lint('source=logs | rex field=m "(?\\d+)"'); }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 1); + assert.equal(out[0].query, 'source=logs | rex field=m "(?\\d+)"'); +}); + +test('template literals with interpolation are dropped', () => { + // `${...}` is filled at runtime; sending the placeholder to the engine tests + // nothing and would be reported as a syntax error. + const source = 'describe(\'field-validation\', () => { lint(`source=${idx} | fields a`); });'; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.equal(out.length, 0); +}); + +test('only PPL-opening strings are treated as queries', () => { + const source = ` + describe('field-validation', () => { + it('x', () => { + expect(msg).toBe('this is not a query at all'); + lint('source=logs | fields a'); + }); + }); + `; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES }); + assert.deepEqual( + out.map((o) => o.query), + ['source=logs | fields a'] + ); +}); + +test('the harvest records where each query came from', () => { + const source = `describe('division-by-zero', () => {\n lint('source=logs | eval x = a / 0');\n});`; + const out = harvestFile(source, { file: 'pkg/x.test.ts', knownRules: RULES }); + assert.match(out[0].source, /^pkg\/x\.test\.ts:2$/); +}); + +// --- index remapping --------------------------------------------------------- + +test('source= is remapped onto the fixture index', () => { + assert.equal( + remapIndex('source=logs | fields a', 'acct'), + 'source=acct | fields a' + ); +}); + +test('a backticked source is remapped', () => { + assert.equal(remapIndex('source=`my-logs` | fields a', 'acct'), 'source=acct | fields a'); +}); + +test('the bare `search ` form is remapped', () => { + assert.equal( + remapIndex('search accounts | eval x = balance / 0', 'acct'), + 'search acct | eval x = balance / 0' + ); +}); + +test('remapping is a no-op without a target index', () => { + assert.equal(remapIndex('source=logs | fields a', ''), 'source=logs | fields a'); +}); + +test('the original query is kept alongside the remapped one', () => { + // Needed to explain a finding: a reader has to be able to see what the OSD test + // actually asserted before trusting a disagreement derived from the rewrite. + const source = `describe('field-validation', () => { lint('source=logs | fields a'); });`; + const out = harvestFile(source, { file: 't.ts', knownRules: RULES, index: 'acct' }); + assert.equal(out[0].query, 'source=acct | fields a'); + assert.equal(out[0].originalQuery, 'source=logs | fields a'); +}); + +test('identifiers are collected for the labeler to intersect against the fixture', () => { + const ids = referencedIdentifiers('source=acct | eval x = durationNano / 0'); + assert.ok(ids.includes('durationNano')); + assert.ok(ids.includes('acct')); +}); + +// --- runner specs ------------------------------------------------------------ + +const CORPUS = { + index: 'acct', + queries: [ + { ruleId: 'division-by-zero', name: 'discovery-0', query: 'source=acct | eval x = a / 0' }, + { ruleId: 'division-by-zero', name: 'discovery-1', query: 'source=acct | eval x = a / 2' }, + { ruleId: 'head-without-sort', name: 'discovery-2', query: 'source=acct | head 5' }, + { ruleId: null, name: 'discovery-3', query: 'source=acct | fields a' }, + ], +}; + +test('one spec is emitted per rule, and unattributed queries are excluded', () => { + const specs = toRunnerSpecs(CORPUS); + assert.deepEqual( + specs.map((s) => s.spec.ruleId), + ['division-by-zero', 'head-without-sort'] + ); + assert.equal(Object.keys(specs[0].spec.queries).length, 2); +}); + +test('generated specs carry no wiring block', () => { + // The runner deep-equals `wiring` against the OSD catalog when present. A + // generated approximation would fail the run for a reason unrelated to discovery. + const specs = toRunnerSpecs(CORPUS); + assert.equal(specs[0].spec.wiring, undefined); +}); + +test('every generated query is declared a trigger', () => { + // Roles are derived later from real detector output. Declaring some as controls + // would make the runner apply control-specific cross-checks whose failures are + // pure noise on a corpus with no pinned verdicts. + const specs = toRunnerSpecs(CORPUS); + for (const spec of specs) { + for (const q of Object.values(spec.spec.queries)) { + assert.equal(q.role, 'trigger'); + } + } +}); + +test('exactly one expectation matches any engine version', () => { + // Two matching entries make the runner report the version as uncovered; an + // open-ended empty range is what guarantees a single match on every leg. + const specs = toRunnerSpecs(CORPUS); + for (const spec of specs) { + assert.equal(spec.spec.expectations.length, 1); + assert.equal(spec.spec.expectations[0].version, ''); + } +}); + +test('generated specs are scored on either grammar surface', () => { + const specs = toRunnerSpecs(CORPUS); + assert.ok(specs.every((s) => s.spec.grammarSurface === 'both')); +}); + +test('query names are preserved so the two halves can be joined', () => { + // The detector runner and the engine probe key on these names. If they disagreed, + // every row would lose its counterpart and the corpus would read as unobserved. + const specs = toRunnerSpecs(CORPUS); + assert.deepEqual(Object.keys(specs[0].spec.queries), ['discovery-0', 'discovery-1']); +}); diff --git a/scripts/ppl-lint/__tests__/label-discovery.test.mjs b/scripts/ppl-lint/__tests__/label-discovery.test.mjs new file mode 100644 index 00000000000..59ca138470e --- /dev/null +++ b/scripts/ppl-lint/__tests__/label-discovery.test.mjs @@ -0,0 +1,242 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for discovery-corpus labelling. + * + * node --test scripts/ppl-lint/__tests__/label-discovery.test.mjs + * + * The labeler turns two observations into a role and, sometimes, a finding. Every + * way it can produce a CONFIDENT finding from a non-observation is a way to send an + * engineer after a bug that does not exist, so the three-state read and the + * uninformative-rejection filter are pinned case by case. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + FINDINGS, + ROLES, + labelQuery, + renderMarkdown, + uninformativeRejection, +} from '../label-discovery.mjs'; + +const base = { ruleId: 'invalid-capture-group-name', query: 'source=acct | rex field=m "(?x)"' }; + +// --- role assignment --------------------------------------------------------- + +test('a query the detector fires on is a trigger', () => { + const row = labelQuery({ ...base, detectorCount: 1, backendRejected: true }); + assert.equal(row.role, ROLES.TRIGGER); +}); + +test('a query the detector ignores is a control', () => { + const row = labelQuery({ ...base, detectorCount: 0, backendRejected: false }); + assert.equal(row.role, ROLES.CONTROL); +}); + +// --- the two findings -------------------------------------------------------- + +test('detector fires + engine accepts is a possible false positive', () => { + const row = labelQuery({ + ...base, + detectorCount: 2, + severities: ['error', 'error'], + backendRejected: false, + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); + assert.match(row.finding.evidence, /ACCEPTED this query/); +}); + +test('an ADVISORY rule the engine accepts is not a false positive', () => { + // Found against a live 3.8 engine: `head-without-sort` and `rex-scan-cost` are + // `info` rules that flag non-determinism and cost. The engine runs those queries + // happily and will never reject them, so "accepted + flagged" is the rule working + // as designed. Without this, every advisory trigger becomes a finding and buries + // the real ones. + const row = labelQuery({ + ...base, + ruleId: 'head-without-sort', + detectorCount: 1, + severities: ['info'], + backendRejected: false, + }); + assert.equal(row.finding, null); + assert.equal(row.advisory, true); + // Still a trigger: it is exactly the kind of trigger the relaxation rollup counts. + assert.equal(row.role, ROLES.TRIGGER); +}); + +test('a mixed-severity diagnostic is not treated as advisory', () => { + // Only an ALL-info diagnostic is advisory. One error-severity marker means the + // rule is asserting the engine will refuse the query, which acceptance contradicts. + const row = labelQuery({ + ...base, + detectorCount: 2, + severities: ['info', 'error'], + backendRejected: false, + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); +}); + +test('an advisory rule the engine REJECTS is still evidence', () => { + // The advisory carve-out applies only to the accepted direction. A silent detector + // on a query the engine refused is unaffected by severity. + const row = labelQuery({ + ...base, + ruleId: 'head-without-sort', + detectorCount: 0, + severities: [], + backendRejected: true, + backendType: 'IllegalArgumentException', + backendReason: 'head requires a positive integer', + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_NEGATIVE); +}); + +test('detector silent + engine rejects is a possible false negative', () => { + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SemanticCheckException', + backendReason: 'capture group name is invalid', + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_NEGATIVE); + // Must tell the reader to verify the rejection is this rule's condition; the + // engine rejecting is weaker evidence than the engine accepting. + assert.match(row.finding.evidence, /VERIFY the rejection is this rule's condition/); +}); + +test('agreement in either direction is not a finding', () => { + assert.equal(labelQuery({ ...base, detectorCount: 1, backendRejected: true }).finding, null); + assert.equal(labelQuery({ ...base, detectorCount: 0, backendRejected: false }).finding, null); +}); + +// --- the three-state read ---------------------------------------------------- + +test('no engine verdict produces no finding', () => { + // The trap this closes: coercing an absent verdict to `false` reads a timed-out + // leg as "the engine accepted this" and manufactures a false-positive finding + // against a healthy rule. + const row = labelQuery({ ...base, detectorCount: 1, backendRejected: undefined }); + assert.equal(row.finding, null); + assert.equal(row.unobserved, true); +}); + +test('a silent detector with no engine verdict is unknown, not a control', () => { + // Calling it a control would claim the rule correctly stayed quiet, which nothing + // observed. It also inflates control counts that the coverage table reports. + const row = labelQuery({ ...base, detectorCount: 0, backendRejected: undefined }); + assert.equal(row.role, ROLES.UNKNOWN); +}); + +// --- uninformative rejections ------------------------------------------------ + +test('an unknown-field rejection is suppressed, not reported', () => { + // Harvested queries name fields their original OSD test invented. Without this + // filter every such query becomes a false-negative finding and buries the real + // ones. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SemanticCheckException', + backendReason: "can't resolve Symbol(namespace=FIELD_NAME, name=durationNano)", + }); + assert.equal(row.finding, null); + assert.equal(row.suppressed, 'unknown field'); +}); + +test('a syntax error is suppressed', () => { + // After index remapping some harvested queries are genuinely malformed; a query + // the grammar cannot parse says nothing about a semantic rule. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendType: 'SyntaxCheckException', + backendReason: 'mismatched input ', + }); + assert.equal(row.suppressed, 'syntax error'); +}); + +test('an unsupported-command rejection is suppressed', () => { + // Same reasoning as the enforced corpus's control-also-rejected guard: the + // command not existing is not evidence about a rule's condition. + const row = labelQuery({ + ...base, + detectorCount: 0, + backendRejected: true, + backendReason: 'union is not supported in this version', + }); + assert.equal(row.suppressed, 'unsupported command'); +}); + +test('an on-topic rejection survives the filter', () => { + assert.equal( + uninformativeRejection('IllegalArgumentException', 'Union command requires at least two datasets'), + null + ); +}); + +test('a rule with no observed trigger is distinguished from one with a single trigger', () => { + // These are different problems: zero triggers means this corpus proves nothing + // about the rule at all (every harvested query was a control, or the detector is + // gated off on this surface), whereas one trigger means the rule is observable but + // a "fully relaxed" verdict would rest on a single case. Rendering both as + // "1 trigger only" hid the first, which is the more serious gap. + const markdown = renderMarkdown({ + stats: { queries: 9, triggers: 4, controls: 5, unknown: 0, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [ + { ruleId: 'none-observed', triggers: 0, controls: 4, sufficientForScopeDecision: false }, + { ruleId: 'single', triggers: 1, controls: 2, sufficientForScopeDecision: false }, + { ruleId: 'plenty', triggers: 3, controls: 2, sufficientForScopeDecision: true }, + ], + }); + assert.match(markdown, /`none-observed` \| 0 \| 4 \| \*\*none — no trigger observed\*\*/); + assert.match(markdown, /`single` \| 1 \| 2 \| \*\*no — 1 trigger only\*\*/); + assert.match(markdown, /`plenty` \| 3 \| 2 \| yes/); +}); + +test('a run with no engine half says so instead of implying agreement', () => { + // "0 finding(s)" beside 109 queries reads as "everything agrees". With no engine + // verdicts nothing was compared at all, and the report has to distinguish those. + const withoutEngine = renderMarkdown({ + differential: false, + stats: { queries: 109, triggers: 19, controls: 0, unknown: 90, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [], + }); + assert.match(withoutEngine, /No engine verdicts were supplied/); + + const withEngine = renderMarkdown({ + differential: true, + stats: { queries: 10, triggers: 4, controls: 6, unknown: 0, findings: 0, suppressed: 0 }, + findings: [], + triggerCoverage: [], + }); + assert.doesNotMatch(withEngine, /No engine verdicts were supplied/); +}); + +test('suppression never applies to the false-POSITIVE side', () => { + // The filter exists to protect the weak (false-negative) direction. A query the + // engine RAN successfully is conclusive regardless of what any error text says, + // so an accepted query must still report even with a suppressible-looking reason. + const row = labelQuery({ + ...base, + detectorCount: 1, + // Explicit rather than relying on the default: with an empty severities list the + // advisory check cannot fire, so the test would pass for the wrong reason and + // stop covering the suppression filter at all. + severities: ['error'], + backendRejected: false, + backendReason: "can't resolve Symbol(name=whatever)", + }); + assert.equal(row.finding.kind, FINDINGS.FALSE_POSITIVE); +}); diff --git a/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs b/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs new file mode 100644 index 00000000000..5368f8afabb --- /dev/null +++ b/scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Unit tests for the discovery engine probe's response mapping. + * + * node --test scripts/ppl-lint/__tests__/probe-discovery-backend.test.mjs + * + * The probe has one job that can go wrong quietly: turning an HTTP response into a + * verdict. Reading a non-answer as acceptance is what converts a network blip into + * "the engine now accepts this query", so the accept/reject/no-verdict split is + * pinned here. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { readResponse } from '../probe-discovery-backend.mjs'; + +test('a 200 is acceptance', () => { + const v = readResponse({ status: 200, bodyText: '{"datarows":[]}' }); + assert.equal(v.outcome, 'observed'); + assert.equal(v.rejected, false); +}); + +test('a 400 is rejection and keeps the engine type and reason', () => { + // The labeler's uninformative-rejection filter keys on these two strings; losing + // them would let an unknown-field rejection be reported as a missed diagnostic. + const v = readResponse({ + status: 400, + bodyText: JSON.stringify({ + error: { type: 'SemanticCheckException', reason: "can't resolve Symbol(name=foo)" }, + }), + }); + assert.equal(v.rejected, true); + assert.equal(v.observed.type, 'SemanticCheckException'); + assert.match(v.observed.reason, /can't resolve/); +}); + +test('a 500 is also rejection', () => { + assert.equal(readResponse({ status: 500, bodyText: '{}' }).rejected, true); +}); + +test('an unparseable body still yields a verdict from the status', () => { + // The engine answered; the body being junk does not change whether it ran the + // query. Discarding the verdict here would lose real signal. + const v = readResponse({ status: 200, bodyText: 'oops' }); + assert.equal(v.outcome, 'observed'); + assert.equal(v.rejected, false); +}); + +test('an enormous reason is truncated', () => { + const v = readResponse({ + status: 400, + bodyText: JSON.stringify({ error: { type: 'X', reason: 'y'.repeat(5000) } }), + }); + assert.equal(v.observed.reason.length, 500); +}); + +test('an accepted response carries no error fields', () => { + const v = readResponse({ status: 200, bodyText: '{"datarows":[]}' }); + assert.equal(v.observed.type, undefined); + assert.equal(v.observed.reason, undefined); +}); diff --git a/scripts/ppl-lint/harvest-queries.mjs b/scripts/ppl-lint/harvest-queries.mjs new file mode 100644 index 00000000000..b641f4612b1 --- /dev/null +++ b/scripts/ppl-lint/harvest-queries.mjs @@ -0,0 +1,576 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Harvest PPL queries out of OSD's own lint test suite into a discovery corpus. + * + * ## Why this exists + * + * The enforced contract corpus is hand-written and therefore small — around one + * trigger and one control per rule. That is right for an enforced contract (every + * expectation is a reviewed claim) but it is too thin to answer the question that + * decides a remediation: when an engine starts accepting a query a rule flags, is + * the behavior FULLY gone on that version, or only PARTIALLY? + * + * `classifyRelaxationScope` in drift.mjs needs several triggers per rule to tell + * those apart, because they need opposite actions — version-scope the rule away + * (full) versus narrow the detector (partial). With one pinned trigger, a partial + * engine fix is indistinguishable from a total one, and the advice that follows + * ships a false negative. + * + * OSD's lint tests already contain that variety: whoever wrote each detector wrote + * several queries that should fire and several that should not, grouped by rule. + * `invalid-capture-group-name` has three distinct reasons a name is invalid + * (hyphen, leading digit, all digits) in OSD's tests versus one in the contract. + * Harvesting them costs nothing and is exactly the input the rollup needs. + * + * ## What this does NOT do + * + * It does not produce expectations, and the discovery corpus never fails a build. + * A harvested query carries no pinned verdict — `label-discovery.mjs` derives its + * role by running the real detectors, and the engine supplies the other half. That + * is deliberate: auto-deriving an expectation from current behavior can only ever + * confirm current behavior, locking in whatever the detector does today, bugs and + * all. Promotion into the enforced corpus stays a human writing a spec entry. + * + * ## Attribution + * + * A query is attributed to a rule by the innermost enclosing `describe('')` + * whose title is a known catalog rule id (OSD's tests are organized that way, see + * `__tests__/silent_failure_rules.test.ts`). A query with no such ancestor is + * recorded with `ruleId: null` and skipped by the labeler unless `--keep-unowned` + * is passed — guessing an owner from a filename would attribute queries to the + * wrong rule, which is worse than dropping them. + * + * Usage: + * node scripts/ppl-lint/harvest-queries.mjs \ + * --osd \ + * --catalog-rules \ + * --index opensearch-sql_test_index_account \ + * --out discovery-corpus.json + */ + +import fs from 'fs'; +import path from 'path'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-harvest] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-harvest] FATAL: ${message}`); + process.exit(2); +} + +/** + * Directories under an OSD checkout that hold PPL lint tests. Kept explicit + * rather than globbing the whole repo: a wide sweep would pull in queries from + * autocomplete/highlighting suites that were never written as lint trigger or + * control cases, and a query harvested from the wrong intent produces a + * "disagreement" that is really just a query nobody claimed anything about. + */ +const LINT_TEST_DIRS = [ + 'packages/osd-monaco/src/ppl/lint/__tests__', + 'packages/osd-monaco/src/ppl/lint/hover/__tests__', + 'packages/osd-monaco/src/ppl/lint/explain/__tests__', +]; + +/** + * Benchmarks and repro captures are excluded. Bench files hold deliberately + * pathological queries built to be slow rather than to be right or wrong, and + * they would dominate the corpus with near-duplicates. + */ +const EXCLUDED_FILE_PATTERNS = [/\.bench\.test\.ts$/, /\.verify\.test\.ts$/]; + +function parseArgs(argv) { + const args = { + osd: '', + out: 'discovery-corpus.json', + index: '', + catalogRules: [], + keepUnowned: false, + maxPerRule: 40, + specsOut: '', + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--osd') args.osd = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--index') args.index = next(); + else if (arg === '--catalog-rules') args.catalogRules = readRuleList(next()); + else if (arg === '--keep-unowned') args.keepUnowned = true; + else if (arg === '--max-per-rule') args.maxPerRule = Number(next()); + else if (arg === '--specs-out') args.specsOut = next(); + else fatal(`unknown argument "${arg}"`); + } + if (!args.osd) fatal('--osd is required'); + return args; +} + +/** Rule ids either inline (`a,b,c`) or from a file (`@path`), one per line or JSON. */ +function readRuleList(value) { + if (!value.startsWith('@')) { + return value + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + } + const file = value.slice(1); + if (!fs.existsSync(file)) fatal(`--catalog-rules file not found: ${file}`); + const raw = fs.readFileSync(file, 'utf8'); + try { + const parsed = JSON.parse(raw); + if (Array.isArray(parsed)) { + // Accept both a bare id list and OSD's rules_catalog.json shape. + return parsed.map((e) => (typeof e === 'string' ? e : e && e.id)).filter(Boolean); + } + } catch { + // not JSON; fall through to line-delimited + } + return raw + .split('\n') + .map((s) => s.trim()) + .filter((s) => s && !s.startsWith('#')); +} + +/** Every lint test file under the OSD checkout, excluding benches. */ +function findTestFiles(osdRoot) { + const files = []; + for (const dir of LINT_TEST_DIRS) { + const abs = path.join(osdRoot, dir); + if (!fs.existsSync(abs)) continue; + for (const name of fs.readdirSync(abs)) { + if (!name.endsWith('.test.ts') && !name.endsWith('.test.tsx')) continue; + if (EXCLUDED_FILE_PATTERNS.some((re) => re.test(name))) continue; + files.push(path.join(abs, name)); + } + } + return files.sort(); +} + +/** + * Track which `describe(...)` blocks enclose a given offset, so a query can be + * attributed to the rule whose block it sits in. + * + * Brace counting is enough here and a real TS parser is not worth the dependency: + * these are test files whose describes are conventional `describe('x', () => {` + * calls. The failure mode of miscounting is a query attributed to an outer block + * (or to none), which the `ruleId: null` path already handles safely — never a + * query attributed to a rule that does not own it, because titles must match a + * known catalog id. + */ +function buildDescribeScopes(source) { + const scopes = []; + const describeRe = /\bdescribe(?:\.\w+)?\s*\(\s*(['"`])((?:\\.|(?!\1).)*)\1/g; + let match; + while ((match = describeRe.exec(source)) !== null) { + const title = match[2]; + // Find the block's opening brace after the describe call, then its matching + // close, ignoring braces inside strings and comments. + const braceStart = source.indexOf('{', match.index + match[0].length); + if (braceStart === -1) continue; + const end = matchBrace(source, braceStart); + scopes.push({ title, start: braceStart, end: end === -1 ? source.length : end }); + } + return scopes; +} + +/** Index of the `}` matching the `{` at `open`, or -1. Skips strings/comments. */ +function matchBrace(source, open) { + let depth = 0; + for (let i = open; i < source.length; i++) { + const ch = source[i]; + if (ch === '/' && source[i + 1] === '/') { + const nl = source.indexOf('\n', i); + i = nl === -1 ? source.length : nl; + continue; + } + if (ch === '/' && source[i + 1] === '*') { + const close = source.indexOf('*/', i + 2); + i = close === -1 ? source.length : close + 1; + continue; + } + if (ch === "'" || ch === '"' || ch === '`') { + i = skipString(source, i); + continue; + } + if (ch === '{') depth++; + else if (ch === '}') { + depth--; + if (depth === 0) return i; + } + } + return -1; +} + +/** Index of the closing quote of the string starting at `start`. */ +function skipString(source, start) { + const quote = source[start]; + for (let i = start + 1; i < source.length; i++) { + if (source[i] === '\\') { + i++; + continue; + } + if (source[i] === quote) return i; + } + return source.length; +} + +/** + * PPL query literals. Anchored on the commands that can open a PPL statement, so + * arbitrary strings in a test file are not mistaken for queries. `search` and + * `source=`/`index=` are the real openers; `describe` is deliberately absent + * because it collides with the test function of the same name. + */ +const QUERY_RE = /(['"`])((?:source\s*=|index\s*=|search\s+)(?:\\.|(?!\1).)*)\1/g; + +/** + * A harvested literal is a JS string literal, so its escapes are JS-level. The + * detectors want the RUNTIME string: `'(?\\\\d+)'` in a test source is + * the four characters `\\d+` on the wire... which is itself a regex escape the + * engine sees. Unescaping the JS layer (and only that layer) is what makes a + * harvested query identical to what the test actually linted. + */ +function unescapeJsString(raw) { + return raw.replace(/\\(u\{[0-9a-fA-F]+\}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|.)/g, (all, esc) => { + switch (esc[0]) { + case 'n': + return '\n'; + case 't': + return '\t'; + case 'r': + return '\r'; + case 'b': + return '\b'; + case 'f': + return '\f'; + case 'v': + return '\v'; + case '0': + return '\0'; + case 'x': + return String.fromCharCode(parseInt(esc.slice(1), 16)); + case 'u': + return esc[1] === '{' + ? String.fromCodePoint(parseInt(esc.slice(2, -1), 16)) + : String.fromCharCode(parseInt(esc.slice(1), 16)); + default: + // Covers \\ \' \" \` and any other single-character escape. + return esc; + } + }); +} + +/** + * Rewrite the test's index onto the backend fixture's index. + * + * OSD's unit tests lint against invented sources (`source=logs`, `search + * accounts`) that do not exist in the SQL integ-test cluster. A harvested query + * must name a real index or the engine rejects it for a reason that has nothing to + * do with the rule — which would read as "the engine rejects this" and be counted + * as a trigger holding. + * + * Only the leading source/index/search clause is rewritten. Subsearch sources + * inside the query body are rewritten too, since those are equally invented. + * Returns null when no rewrite was possible, so the caller can drop the query + * rather than send an unresolvable index to the cluster. + */ +export function remapIndex(query, targetIndex) { + if (!targetIndex) return query; + let out = query + .replace(/\bsource\s*=\s*`[^`]+`/g, `source=${targetIndex}`) + .replace(/\bsource\s*=\s*[A-Za-z_][\w.*-]*/g, `source=${targetIndex}`) + .replace(/\bindex\s*=\s*`[^`]+`/g, `index=${targetIndex}`) + .replace(/\bindex\s*=\s*[A-Za-z_][\w.*-]*/g, `index=${targetIndex}`); + // `search ` — the bare-index form. Only the opener, and only when the + // token is not already a keyword-led clause. + out = out.replace(/^(\s*search\s+)(?!source\s*=|index\s*=)([A-Za-z_][\w.*-]*)/, `$1${targetIndex}`); + return out; +} + +/** + * Does this query reference fields the backend fixture will not have? + * + * A harvested query naming `durationNano` against the `account` index is rejected + * for an unknown field, not for the rule's condition. Counting that as "the engine + * still rejects" would fake a partial fix and send someone to narrow a healthy + * detector — the same class of vacuous result the enforced contract's + * control-also-rejected guard exists to prevent. + * + * This cannot be decided statically, so it is not decided here: the query is + * harvested with the field names it mentions recorded, and the labeler drops the + * ones the fixture cannot satisfy. Extracting identifiers is best-effort and + * deliberately over-broad (it will include command keywords), because the labeler + * intersects against the fixture's real field list rather than trusting this. + */ +export function referencedIdentifiers(query) { + const ids = new Set(); + for (const match of query.matchAll(/\b([A-Za-z_][\w.]*)\b/g)) { + ids.add(match[1]); + } + return [...ids]; +} + +/** + * The rule a `describe(...)` title names, or null. + * + * OSD's titles are not bare ids — a rule-scoped suite reads + * `describe('rex-scan-cost (compiled surface)')` or + * `describe('field-validation alternate-source suppression')`. Requiring an exact + * match dropped ~75% of harvestable queries, all of them from files dedicated to a + * single rule, so the title is matched as a PREFIX at a word boundary. + * + * Prefix-only is the point: matching a rule id anywhere in the title would let + * `describe('does not fire on rex-scan-cost candidates')`, nested under a + * different rule, steal the attribution. A title that merely mentions another rule + * mid-sentence is not that rule's suite. Longest match wins, so + * `field-validation-shape` is preferred over `field-validation` when both exist. + */ +export function ruleFromDescribeTitle(title, knownRules) { + const text = String(title || ''); + let best = null; + for (const ruleId of knownRules) { + if (!text.startsWith(ruleId)) continue; + // Must end at a word boundary: `head-without-sorting` is not `head-without-sort`. + const after = text.charAt(ruleId.length); + if (after && /[\w-]/.test(after)) continue; + if (!best || ruleId.length > best.length) best = ruleId; + } + return best; +} + +/** Harvest one file into `{ ruleId, query, source }` records. */ +export function harvestFile(source, { file, knownRules, index }) { + const scopes = buildDescribeScopes(source); + const known = new Set(knownRules || []); + const out = []; + for (const match of source.matchAll(QUERY_RE)) { + const raw = match[2]; + const query = unescapeJsString(raw); + // Template literals with interpolation are not real queries — the `${...}` is + // a placeholder the test fills at runtime, and sending it to the engine tests + // nothing. Dropped rather than guessed at. + if (/\$\{/.test(query)) continue; + // A query must have at least one pipe or be a bare source read; anything + // shorter is usually a fragment asserted against, not a lintable statement. + if (query.trim().length < 8) continue; + + // Innermost enclosing describe whose title is a known rule id. + const at = match.index; + const enclosing = scopes + .filter((s) => at > s.start && at < s.end) + .sort((a, b) => b.start - a.start); + // Innermost first: a query inside `describe('flat-object-subfield')` nested in + // `describe('silent-failure rules')` belongs to the specific rule, not the file. + let owner = null; + for (const scope of enclosing) { + owner = ruleFromDescribeTitle(scope.title, known); + if (owner) break; + } + const line = source.slice(0, at).split('\n').length; + + out.push({ + ruleId: owner, + query: index ? remapIndex(query, index) : query, + originalQuery: query, + identifiers: referencedIdentifiers(query), + source: `${file}:${line}`, + }); + } + return out; +} + +/** + * Emit the harvested corpus as spec files the EXISTING detector runner can consume. + * + * `run-frontend-contract.mjs` is expectation-driven: it walks `expectations[]`, + * scores each query against a pinned `detectorCount`, and records the real `actual` + * count in its report either way. Discovery needs only that `actual`, so rather + * than teach the runner a second mode — which would risk changing how the ENFORCED + * check behaves — the corpus is written out as ordinary specs whose expectations are + * deliberately arbitrary. + * + * Two consequences, both intended: + * - The runner will report failures for every query whose real count differs from + * the placeholder. Those are meaningless here and the caller discards the exit + * code; only `detector-report.json` is read. This is why discovery must never + * be wired to a required check. + * - `grammarSurface: 'both'` so a rule is scored on whichever surface the leg + * ran, and `schedule: 'nightly'` to match how the aggregate legs invoke it. + * + * One spec per rule, because the runner keys wiring checks off `spec.ruleId`. + */ +export function toRunnerSpecs(corpus) { + const byRule = new Map(); + for (const [i, entry] of (corpus.queries || []).entries()) { + if (!entry.ruleId) continue; + if (!byRule.has(entry.ruleId)) byRule.set(entry.ruleId, []); + byRule.get(entry.ruleId).push({ ...entry, name: entry.name || `discovery-${i}` }); + } + + const specs = []; + for (const [ruleId, entries] of [...byRule].sort()) { + const queries = {}; + const expected = {}; + for (const entry of entries) { + // Every query is declared a trigger: the runner needs SOME role, and the real + // role is derived later from the detector's actual output. Calling them all + // triggers keeps the runner from applying its control-specific cross-checks, + // whose failures would be pure noise on a corpus with no pinned verdicts. + queries[entry.name] = { role: 'trigger', query: entry.query }; + expected[entry.name] = { detectorCount: 0 }; + } + specs.push({ + fileName: `${ruleId}.discovery.spec.json`, + spec: { + schemaVersion: 3, + ruleId, + grammarSurface: 'both', + schedule: 'nightly', + // No `wiring` block: the runner deep-equals it against the catalog when + // present, and a mismatch there would fail the run for a reason that has + // nothing to do with discovery. + index: corpus.index || undefined, + queries, + // A single open expectation so exactly one entry matches every engine + // version; the pinned counts are placeholders (see the note above). + expectations: [{ version: '', queries: expected }], + }, + }); + } + return specs; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const files = findTestFiles(args.osd); + if (files.length === 0) { + fatal(`no lint test files found under ${args.osd}; is --osd an OSD checkout root?`); + } + + const records = []; + for (const file of files) { + const source = fs.readFileSync(file, 'utf8'); + const rel = path.relative(args.osd, file); + records.push( + ...harvestFile(source, { file: rel, knownRules: args.catalogRules, index: args.index }) + ); + } + + // Dedupe on (ruleId, query): the same query legitimately appears in several + // tests, and running it repeatedly against the cluster buys nothing. + const seen = new Set(); + const unique = []; + for (const record of records) { + const key = `${record.ruleId}::${record.query}`; + if (seen.has(key)) continue; + seen.add(key); + unique.push(record); + } + + const owned = unique.filter((r) => r.ruleId); + const unowned = unique.filter((r) => !r.ruleId); + + // Cap per rule so one heavily-tested rule cannot dominate a leg's runtime. The + // drop is LOGGED rather than silent: a truncated corpus that reads as complete + // is how a coverage gap hides. + const byRule = new Map(); + for (const record of owned) { + if (!byRule.has(record.ruleId)) byRule.set(record.ruleId, []); + byRule.get(record.ruleId).push(record); + } + const kept = []; + for (const [ruleId, group] of [...byRule].sort()) { + if (group.length > args.maxPerRule) { + log( + `NOTE: ${ruleId} harvested ${group.length} queries; keeping the first ${args.maxPerRule} ` + + `(--max-per-rule). ${group.length - args.maxPerRule} dropped.` + ); + } + kept.push(...group.slice(0, args.maxPerRule)); + } + + // Names are assigned ONCE, here, and every downstream artifact keys off them. If + // the detector runner and the engine probe derived names independently they could + // disagree, and every row would silently lose its counterpart — the whole corpus + // would read as unobserved rather than as a bug. + kept.forEach((entry, i) => { + entry.name = `discovery-${i}`; + }); + + const corpus = { + schemaVersion: 1, + kind: 'discovery', + // Stated in the artifact itself so no downstream consumer can mistake this for + // the enforced corpus and start failing builds on it. + enforced: false, + note: + 'Auto-harvested from OSD lint tests. Roles are assigned by label-discovery.mjs from real ' + + 'detector output; there are no pinned expectations and this corpus must never fail a build.', + index: args.index || null, + sourceFiles: files.map((f) => path.relative(args.osd, f)), + queries: kept, + unowned: args.keepUnowned ? unowned : [], + stats: { + files: files.length, + harvested: records.length, + unique: unique.length, + owned: kept.length, + unowned: unowned.length, + rules: byRule.size, + }, + }; + + fs.writeFileSync(args.out, JSON.stringify(corpus, null, 2)); + log( + `wrote ${args.out}: ${kept.length} owned query(s) across ${byRule.size} rule(s) from ` + + `${files.length} file(s); ${unowned.length} unattributed` + + (args.keepUnowned ? ' (kept)' : ' (dropped)') + ); + + // Optional: the same corpus as spec files, so the existing detector runner can + // produce real diagnostic counts for it without being modified. + if (args.specsOut) { + fs.mkdirSync(args.specsOut, { recursive: true }); + const specs = toRunnerSpecs(corpus); + for (const { fileName, spec } of specs) { + fs.writeFileSync(path.join(args.specsOut, fileName), JSON.stringify(spec, null, 2)); + } + fs.writeFileSync( + path.join(args.specsOut, 'manifest.json'), + JSON.stringify( + { + schemaVersion: 3, + description: + 'AUTO-GENERATED discovery corpus. No reviewed expectations; the pinned counts are ' + + 'placeholders. Never list these under defaultError and never wire them to a required check.', + contracts: specs.map((s) => s.fileName), + // Empty on purpose: `defaultError` is the ENFORCED set, and nothing here is + // enforced. A non-empty value would make the aggregator fail the build on + // auto-generated expectations. + defaultError: [], + }, + null, + 2 + ) + ); + log(`wrote ${specs.length} runner spec(s) to ${args.specsOut}`); + } + for (const [ruleId, group] of [...byRule].sort()) { + log(` ${ruleId}: ${Math.min(group.length, args.maxPerRule)}`); + } +} + +// Importable for unit tests; only runs the CLI when executed directly. +if (process.argv[1] && path.resolve(process.argv[1]).endsWith('harvest-queries.mjs')) { + main(); +} diff --git a/scripts/ppl-lint/label-discovery.mjs b/scripts/ppl-lint/label-discovery.mjs new file mode 100644 index 00000000000..3ee6c318860 --- /dev/null +++ b/scripts/ppl-lint/label-discovery.mjs @@ -0,0 +1,467 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Assign roles to harvested queries and report detector/engine disagreements. + * + * ## Why roles are derived, not authored + * + * A trigger is a query the rule's detector fires on; a control is one it stays + * silent on. That is mechanically checkable, so it is checked rather than declared: + * the enforced corpus's hand-set `role` field carries a reviewer's intent, but for + * a harvested corpus of 100+ queries hand-labelling is both the bottleneck and a + * source of error. This script reads the detector report the OSD runner already + * produces and labels from it. + * + * Deriving roles is safe. Deriving EXPECTATIONS would not be: an expectation + * auto-set from current behavior can only ever confirm current behavior, locking in + * whatever the detector does today including its bugs. So this script pins nothing + * and never fails a build. It emits findings. + * + * ## The finding it exists for + * + * With both halves observed, the interesting cell needs no expected output at all: + * + * detector engine meaning + * silent rejects possible FALSE NEGATIVE + * fires accepts possible FALSE POSITIVE + * fires rejects agreement + * silent accepts agreement + * + * "Possible", not "confirmed", and the asymmetry is deliberate. A false positive + * is nearly conclusive: the engine ran the query fine and the linter called it + * broken. A false negative is much weaker — the engine may have rejected the query + * for a reason that has nothing to do with this rule (an unknown field, an index + * that does not exist, a command the version predates), in which case the linter + * was right to stay quiet. Both are reported, ranked, and neither is ever asserted. + * + * ## What this feeds + * + * `classifyRelaxationScope` needs several triggers per rule to tell a FULL engine + * fix (version-scope the rule away) from a PARTIAL one (narrow the detector). This + * corpus is where that trigger variety comes from. For that use it needs only + * "does any trigger still get rejected" — a single counterexample settles the + * question, which is why no pinned verdict is required. + * + * Usage: + * node scripts/ppl-lint/label-discovery.mjs \ + * --corpus discovery-corpus.json \ + * --detector discovery-detector-report.json \ + * --backend discovery-backend-report.json \ + * --fixture-fields account-fields.json \ + * --out discovery-findings.json [--summary $GITHUB_STEP_SUMMARY] + */ + +import fs from 'fs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-discovery] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-discovery] FATAL: ${message}`); + process.exit(2); +} + +/** Roles a harvested query can be assigned. */ +export const ROLES = { + TRIGGER: 'trigger', + CONTROL: 'control', + UNKNOWN: 'unknown', +}; + +/** Finding kinds, ranked by how conclusive they are. */ +export const FINDINGS = { + FALSE_POSITIVE: 'possible-false-positive', + FALSE_NEGATIVE: 'possible-false-negative', +}; + +function parseArgs(argv) { + const args = { + corpus: '', + detector: '', + backend: '', + fixtureFields: '', + out: 'discovery-findings.json', + summary: '', + version: '', + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--corpus') args.corpus = next(); + else if (arg === '--detector') args.detector = next(); + else if (arg === '--backend') args.backend = next(); + else if (arg === '--fixture-fields') args.fixtureFields = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--summary') args.summary = next(); + else if (arg === '--version') args.version = next(); + else fatal(`unknown argument "${arg}"`); + } + if (!args.corpus) fatal('--corpus is required'); + if (!args.detector) fatal('--detector is required'); + return args; +} + +function readJson(file, { optional = false } = {}) { + if (!fs.existsSync(file)) { + if (optional) return undefined; + fatal(`expected file not found: ${file}`); + } + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + if (optional) return undefined; + fatal(`could not parse ${file}: ${error.message}`); + } + return undefined; +} + +/** + * Reasons an engine rejection tells us nothing about the rule under test. + * + * This is the guard that keeps the false-negative side honest. A harvested query + * mentions whatever fields its original OSD unit test invented, and those rarely + * exist in the SQL integ-test index. The engine then rejects for an unknown field + * — and reading that as "the engine rejects this, so the silent detector is a false + * negative" would generate a finding per harvested query and bury the real ones. + * + * Also excluded: a command the engine version does not have. Same logic as the + * enforced corpus's control-also-rejected guard — "unsupported command" is not + * evidence about a rule's specific condition. + */ +export const UNINFORMATIVE_REJECTION_PATTERNS = [ + { re: /can't resolve|cannot resolve|unknown field|no such field|field \[[^\]]+\] not found/i, why: 'unknown field' }, + { re: /IndexNotFoundException|no such index|index \[[^\]]+\] (does not exist|not found)/i, why: 'missing index' }, + { re: /unsupported (command|operation)|is not supported|not yet supported/i, why: 'unsupported command' }, + { re: /SyntaxCheckException|ParseException|mismatched input|extraneous input/i, why: 'syntax error' }, +]; + +/** + * Is this rejection informative about the rule, or an artifact of the harvested + * query not fitting the fixture? Returns the reason it is uninformative, or null. + * + * A syntax error counts as uninformative on purpose. A harvested query that the + * grammar cannot even parse says nothing about a semantic rule — and after index + * remapping some harvested queries genuinely are malformed (a join whose right-hand + * index was a bare identifier the remap could not reach). Treating those as + * evidence would be the vacuous-finding equivalent of a timed-out leg. + */ +export function uninformativeRejection(backendType, backendReason) { + const text = `${backendType || ''} ${backendReason || ''}`; + for (const { re, why } of UNINFORMATIVE_REJECTION_PATTERNS) { + if (re.test(text)) return why; + } + return null; +} + +/** + * Label one query and decide whether it is a finding. + * + * `detectorCount` and `backendRejected` come from the two observation halves. + * `backendRejected === undefined` means no verdict arrived, which is a third state: + * the query is labelled but produces no finding, because a leg that did not answer + * must never generate linter advice. + */ +export function labelQuery({ + ruleId, + query, + detectorCount, + backendRejected, + backendType, + backendReason, + severities = [], +}) { + const fired = (detectorCount || 0) > 0; + // An advisory diagnostic is one the engine will never contradict: `info` severity + // marks cost or non-determinism, not an error the engine would refuse. Read from + // the severities the detector actually EMITTED rather than from the catalog, so a + // rule that emits mixed severities is judged on what this query produced. + const advisory = fired && severities.length > 0 && severities.every((s) => s === 'info'); + const role = fired ? ROLES.TRIGGER : ROLES.CONTROL; + const base = { + ruleId, + query, + role, + detectorCount: detectorCount || 0, + backendRejected, + severities, + }; + + // No engine verdict: label the role (which only needs the detector) but claim + // nothing about correctness. + if (typeof backendRejected !== 'boolean') { + return { ...base, role: fired ? ROLES.TRIGGER : ROLES.UNKNOWN, finding: null, unobserved: true }; + } + + // Detector fires, engine accepts → the linter called a working query broken. + // Nearly conclusive: nothing about the fixture can make a query the engine RAN + // into a rule violation. + // + // EXCEPT for advisory rules. `head-without-sort` (info) and `rex-scan-cost` (info) + // flag non-determinism and cost — things the engine executes happily and will + // never reject. For those, "engine accepts + detector fires" is the rule working + // exactly as designed, not a false positive. Without this the report is dominated + // by every advisory rule's every trigger, and the real findings are unreadable. + // + // Severity is the discriminator because it already encodes the distinction: an + // error/warning rule asserts the engine will refuse or mishandle the query, and + // only such a claim can be contradicted by the engine accepting it. + if (fired && backendRejected === false) { + if (advisory) { + return { ...base, finding: null, advisory: true }; + } + return { + ...base, + finding: { + kind: FINDINGS.FALSE_POSITIVE, + evidence: + `the engine ACCEPTED this query but "${ruleId}" emitted ${detectorCount} diagnostic(s). ` + + `A user running this query sees an error marker on a query that works.`, + }, + }; + } + + // Detector silent, engine rejects → possible missed diagnostic, but only if the + // rejection is about something this rule could have caught. + if (!fired && backendRejected === true) { + const uninformative = uninformativeRejection(backendType, backendReason); + if (uninformative) { + return { ...base, finding: null, suppressed: uninformative }; + } + return { + ...base, + finding: { + kind: FINDINGS.FALSE_NEGATIVE, + evidence: + `the engine REJECTED this query (${backendType || 'error'}: ${backendReason || 'no reason'}) ` + + `and "${ruleId}" stayed silent. VERIFY the rejection is this rule's condition before acting — ` + + `a rejection for an unrelated reason is not a missed diagnostic.`, + }, + }; + } + + return { ...base, finding: null }; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const corpus = readJson(args.corpus); + const detectorReport = readJson(args.detector); + const backendReport = readJson(args.backend, { optional: true }); + + if (corpus.enforced) { + // A corpus that claims to be enforced does not belong on this path: this script + // pins nothing and exits zero, so running it over the enforced corpus would + // look like validation while asserting nothing. + fatal('--corpus is marked enforced; this script only labels the discovery corpus.'); + } + + const detectorByKey = new Map(); + for (const row of detectorReport.results || []) { + detectorByKey.set(`${row.ruleId}::${row.queryName}`, row); + } + const backendByKey = new Map(); + for (const row of Array.isArray(backendReport) ? backendReport : []) { + backendByKey.set(`${row.ruleId}::${row.queryName}`, row); + } + + const labelled = []; + for (const [i, entry] of (corpus.queries || []).entries()) { + const queryName = entry.name || `discovery-${i}`; + const key = `${entry.ruleId}::${queryName}`; + const detectorRow = detectorByKey.get(key); + const backendRow = backendByKey.get(key); + // Same three-state read as the enforced aggregator: `outcome: "error"` carries + // no verdict, and coercing that absence to "accepted" would manufacture a + // false-positive finding out of a network blip. + const hasVerdict = + !!backendRow && + backendRow.outcome !== 'error' && + (typeof backendRow.rejected === 'boolean' || !!backendRow.observed); + + labelled.push({ + ...labelQuery({ + ruleId: entry.ruleId, + query: entry.query, + detectorCount: detectorRow ? detectorRow.actual : undefined, + severities: (detectorRow && detectorRow.severities) || [], + backendRejected: hasVerdict ? !!backendRow.rejected : undefined, + backendType: backendRow && backendRow.observed ? backendRow.observed.type : undefined, + backendReason: backendRow && backendRow.observed ? backendRow.observed.reason : undefined, + }), + queryName, + source: entry.source, + noDetectorRow: !detectorRow, + }); + } + + const findings = labelled.filter((l) => l.finding); + const byRule = new Map(); + for (const row of labelled) { + if (!byRule.has(row.ruleId)) { + byRule.set(row.ruleId, { triggers: [], controls: [], unknown: [], suppressed: 0 }); + } + const bucket = byRule.get(row.ruleId); + if (row.suppressed) bucket.suppressed++; + if (row.role === ROLES.TRIGGER) bucket.triggers.push(row.queryName); + else if (row.role === ROLES.CONTROL) bucket.controls.push(row.queryName); + else bucket.unknown.push(row.queryName); + } + + const report = { + schemaVersion: 1, + kind: 'discovery-findings', + enforced: false, + engineVersion: args.version || detectorReport.engineVersion || null, + surface: detectorReport.surface || null, + // Whether an engine half was supplied at all. Without it the run yields trigger + // counts but structurally cannot yield findings, and the report has to say which + // of those two it is. + differential: backendByKey.size > 0, + stats: { + queries: labelled.length, + triggers: labelled.filter((l) => l.role === ROLES.TRIGGER).length, + controls: labelled.filter((l) => l.role === ROLES.CONTROL).length, + unknown: labelled.filter((l) => l.role === ROLES.UNKNOWN).length, + suppressed: labelled.filter((l) => l.suppressed).length, + // Advisory triggers the engine accepted. Counted so the number is visible: it + // is the single largest category the filter removes, and a silent removal + // would make the corpus look smaller than it is. + advisory: labelled.filter((l) => l.advisory).length, + findings: findings.length, + falsePositives: findings.filter((f) => f.finding.kind === FINDINGS.FALSE_POSITIVE).length, + falseNegatives: findings.filter((f) => f.finding.kind === FINDINGS.FALSE_NEGATIVE).length, + }, + // Per-rule trigger counts are the payload the relaxation rollup consumes: a + // rule with several triggers can distinguish a partial engine fix from a full + // one, and a rule with one cannot. + triggerCoverage: [...byRule] + .map(([ruleId, b]) => ({ + ruleId, + triggers: b.triggers.length, + controls: b.controls.length, + unknown: b.unknown.length, + suppressed: b.suppressed, + // Below two triggers, "every trigger relaxed" is a single observation and + // cannot support a version-scoping decision. Flagged so the gap is visible + // rather than implied by a number nobody reads. + sufficientForScopeDecision: b.triggers.length >= 2, + })) + .sort((a, b) => a.ruleId.localeCompare(b.ruleId)), + findings: findings.map((f) => ({ + ruleId: f.ruleId, + kind: f.finding.kind, + evidence: f.finding.evidence, + query: f.query, + source: f.source, + })), + labelled, + }; + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + log(`wrote ${args.out}`); + + const markdown = renderMarkdown(report); + // eslint-disable-next-line no-console + console.log(markdown); + if (args.summary) { + try { + fs.appendFileSync(args.summary, markdown + '\n'); + } catch (error) { + log(`WARN: could not write summary to ${args.summary}: ${error.message}`); + } + } + + // ALWAYS exit zero. This corpus has no reviewed expectations, so a finding here + // is a lead to investigate, not a proven defect — failing the build on it would + // block unrelated PRs on the strength of an auto-generated guess. + log( + `discovery: ${report.stats.findings} finding(s) from ${report.stats.queries} query(s) ` + + `(${report.stats.falsePositives} possible false positive(s), ` + + `${report.stats.falseNegatives} possible false negative(s)); ` + + `${report.stats.suppressed} rejection(s) suppressed as uninformative, ` + + `${report.stats.advisory} advisory trigger(s) excluded. Not enforced.` + ); +} + +export function renderMarkdown(report) { + const lines = []; + lines.push('## PPL lint discovery corpus (not enforced)'); + lines.push(''); + lines.push( + `Engine \`${report.engineVersion || 'unknown'}\`${report.surface ? ` (${report.surface})` : ''} — ` + + `${report.stats.queries} harvested query(s): ${report.stats.triggers} trigger, ` + + `${report.stats.controls} control, ${report.stats.unknown} unknown. ` + + `**${report.stats.findings} finding(s)** — these are LEADS, not failures.` + ); + lines.push(''); + // Without an engine half there is nothing to disagree WITH, so the run can only + // count triggers. Saying so beats printing "0 finding(s)" next to a large corpus, + // which reads as "everything agrees" when in fact nothing was compared. + if (!report.differential) { + lines.push( + '> No engine verdicts were supplied, so no agreement was checked and no finding can be ' + + 'produced. Trigger counts below are still valid — they come from the detector alone.' + ); + lines.push(''); + } + + if (report.stats.findings > 0) { + lines.push('### Findings'); + lines.push(''); + // False positives first: a query the engine ran successfully but the linter + // marked broken is nearly conclusive, while a false negative may be a rejection + // for an unrelated reason. + const order = [FINDINGS.FALSE_POSITIVE, FINDINGS.FALSE_NEGATIVE]; + for (const kind of order) { + const group = report.findings.filter((f) => f.kind === kind); + if (group.length === 0) continue; + lines.push(`#### ${kind} (${group.length})`); + for (const f of group) { + lines.push(`- \`${f.ruleId}\`: ${f.evidence}`); + lines.push(` QUERY: \`${f.query}\``); + if (f.source) lines.push(` HARVESTED FROM: ${f.source}`); + } + lines.push(''); + } + } + + lines.push('### Trigger coverage'); + lines.push(''); + lines.push('Whether each rule has enough triggers to tell a PARTIAL engine fix from a FULL one.'); + lines.push(''); + lines.push('| Rule | Triggers | Controls | Enough for a scope decision? |'); + lines.push('| ---- | -------- | -------- | ---------------------------- |'); + for (const row of report.triggerCoverage) { + // Zero triggers and one trigger are different problems and must not read the + // same. No trigger at all means this corpus proves nothing about the rule — + // usually that the harvested queries are all controls, or that the detector + // never fired because it is gated off on this surface. One trigger means the + // rule is observable but a "fully relaxed" verdict would rest on a single case. + let verdict; + if (row.triggers === 0) { + verdict = '**none — no trigger observed**'; + } else if (row.triggers === 1) { + verdict = '**no — 1 trigger only**'; + } else { + verdict = 'yes'; + } + lines.push(`| \`${row.ruleId}\` | ${row.triggers} | ${row.controls} | ${verdict} |`); + } + lines.push(''); + return lines.join('\n'); +} + +// Importable for unit tests; only runs the CLI when executed directly. +if (process.argv[1] && process.argv[1].endsWith('label-discovery.mjs')) { + main(); +} diff --git a/scripts/ppl-lint/probe-discovery-backend.mjs b/scripts/ppl-lint/probe-discovery-backend.mjs new file mode 100644 index 00000000000..8ebb0d25d75 --- /dev/null +++ b/scripts/ppl-lint/probe-discovery-backend.mjs @@ -0,0 +1,196 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +/** + * Run the discovery corpus against a live engine and record each verdict. + * + * This is the engine half of the discovery pipeline, and it deliberately does NOT + * go through `PplLintRuleValidationIT`. The IT is an assertion harness built around + * reviewed contract files: it selects a pinned expectation per query and compares + * against it. Discovery queries have no pinned expectation by design, so there is + * nothing for the IT to assert and no reason to pay for a Gradle test-cluster run. + * All that is needed is `POST /_plugins/_ppl` per query and the verdict recorded — + * which is what this does, in the same report shape the aggregator and the labeler + * already read. + * + * Emits `[{ ruleId, queryName, rejected, outcome, observed: { httpStatus, type, + * reason } }]`, matching `backend-report.json` so `label-discovery.mjs` can read + * either source without a special case. + * + * The `outcome` field carries the distinction everything downstream depends on: + * + * observed the engine answered; `rejected` is a real verdict + * error no answer arrived (timeout, connection refused, unparseable body) + * + * Never collapse `error` into `rejected: false`. That coercion is what turns a + * network blip into "the engine now ACCEPTS this query" and generates a + * false-positive finding against a healthy rule. + * + * Usage: + * node scripts/ppl-lint/probe-discovery-backend.mjs \ + * --corpus discovery-corpus.json \ + * --endpoint http://localhost:9200 \ + * --out discovery-backend-report.json [--timeout-ms 15000] [--concurrency 4] + */ + +import fs from 'fs'; + +function log(message) { + // eslint-disable-next-line no-console + console.log(`[ppl-lint-probe] ${message}`); +} + +function fatal(message) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-probe] FATAL: ${message}`); + process.exit(2); +} + +function parseArgs(argv) { + const args = { + corpus: '', + endpoint: 'http://localhost:9200', + out: 'discovery-backend-report.json', + timeoutMs: 15000, + concurrency: 4, + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const next = () => { + const value = argv[++i]; + if (value === undefined) fatal(`${arg} requires a value`); + return value; + }; + if (arg === '--corpus') args.corpus = next(); + else if (arg === '--endpoint') args.endpoint = next(); + else if (arg === '--out') args.out = next(); + else if (arg === '--timeout-ms') args.timeoutMs = Number(next()); + else if (arg === '--concurrency') args.concurrency = Math.max(1, Number(next())); + else fatal(`unknown argument "${arg}"`); + } + if (!args.corpus) fatal('--corpus is required'); + return args; +} + +/** + * Read one PPL response into a verdict. + * + * A 2xx is acceptance. A 4xx/5xx is rejection, and the engine's `error.type` / + * `error.reason` are extracted because the labeler's uninformative-rejection filter + * keys on them — a rejection for an unknown field must not be read as evidence + * about a lint rule. + * + * Exported so the mapping is unit-testable without a cluster. + */ +export function readResponse({ status, bodyText }) { + let body; + try { + body = bodyText ? JSON.parse(bodyText) : undefined; + } catch { + body = undefined; + } + const error = (body && body.error) || {}; + const rejected = status >= 400; + return { + outcome: 'observed', + rejected, + observed: { + httpStatus: status, + rejected, + ...(rejected + ? { + type: error.type || undefined, + // Truncated: engine reasons can embed a whole stack trace, and the full + // text bloats the report without adding signal for the filter. + reason: typeof error.reason === 'string' ? error.reason.slice(0, 500) : undefined, + } + : {}), + }, + }; +} + +/** One query against the engine. Never throws: a failure becomes `outcome: error`. */ +async function probeOne({ endpoint, query, timeoutMs }) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(`${endpoint.replace(/\/$/, '')}/_plugins/_ppl`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ query }), + signal: controller.signal, + }); + const bodyText = await response.text(); + return readResponse({ status: response.status, bodyText }); + } catch (error) { + // No verdict. Recorded as such rather than guessed at — see the header note. + return { + outcome: 'error', + observed: undefined, + error: String((error && error.message) || error), + }; + } finally { + clearTimeout(timer); + } +} + +/** Run `tasks` with at most `limit` in flight, preserving input order. */ +async function mapLimit(items, limit, fn) { + const results = new Array(items.length); + let next = 0; + const workers = Array.from({ length: Math.min(limit, items.length) }, async () => { + for (;;) { + const index = next++; + if (index >= items.length) return; + results[index] = await fn(items[index], index); + } + }); + await Promise.all(workers); + return results; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const corpus = JSON.parse(fs.readFileSync(args.corpus, 'utf8')); + const queries = corpus.queries || []; + if (queries.length === 0) fatal(`corpus ${args.corpus} has no queries`); + + log(`probing ${queries.length} query(s) against ${args.endpoint} (concurrency ${args.concurrency})`); + + const report = await mapLimit(queries, args.concurrency, async (entry, i) => { + const verdict = await probeOne({ + endpoint: args.endpoint, + query: entry.query, + timeoutMs: args.timeoutMs, + }); + return { + ruleId: entry.ruleId, + // Must match the name `label-discovery.mjs` derives, or every row misses its + // detector counterpart and the whole corpus reads as unobserved. + queryName: entry.name || `discovery-${i}`, + role: 'discovery', + query: entry.query, + ...verdict, + }; + }); + + fs.writeFileSync(args.out, JSON.stringify(report, null, 2)); + const errors = report.filter((r) => r.outcome === 'error').length; + const rejected = report.filter((r) => r.rejected === true).length; + log( + `wrote ${args.out}: ${report.length} probed, ${rejected} rejected, ` + + `${report.length - rejected - errors} accepted, ${errors} unobserved` + ); + if (errors > 0) { + // A warning, not a failure: discovery is best-effort and the labeler already + // withholds findings for unobserved queries. Saying nothing would let a leg + // that mostly failed look like a leg that mostly agreed. + log(`WARN: ${errors} query(s) produced no verdict; those yield no findings.`); + } +} + +if (process.argv[1] && process.argv[1].endsWith('probe-discovery-backend.mjs')) { + main().catch((error) => fatal(String((error && error.stack) || error))); +} From 2ec07ac46fb498271db40c560b738368916b6384 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:35:23 -0700 Subject: [PATCH 20/39] feat(ci): harvest each test file's lint context so context-gated rules produce triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Harvesting queries without the context they were written against left 6 of 12 rules at ZERO triggers. All six are `needsContext: true` — they self-suppress without a `typeMap`, so the detector never ran. In the report that is indistinguishable from a rule that fired on nothing, and it is the state the trigger-coverage table exists to surface: `rex-scan-cost` contributed 26 queries and no triggers at all. The context now comes from the same test file as the queries — its `typeMap` and `disabledObjectFields` — because the OSD author wrote it to make exactly those queries fire. A hand-written substitute would be a guess about which field types each query depends on, and a wrong guess silently suppresses the detector again. A rule tested under two different contexts gets two spec files; merging them would hand a query field types its own test never used, so the verdict would describe a scenario nobody wrote. Two further fixes found by running the pipeline rather than by reading it: - `visibleIndices` is now supplied unconditionally, not only when a typeMap exists. `wildcard-source-zero-match` reads only that list and self-suppresses when it is empty, and its test file declares no typeMap — so keying it off the mapping left the rule permanently inert. - A wildcard source is no longer remapped. Rewriting `source=`nope-*`` onto the fixture index destroyed the only thing that rule detects, turning its one harvested query into a control. Verified against a live 3.8 cluster: triggers 19 -> 41, rules with enough triggers to support a scope decision 5 -> 9 of 12, still 0 findings. The three rules left at one trigger are at the ceiling of what OSD's tests contain — their remaining queries are genuine controls (`stats avg(balance)` on a numeric is valid; `row_number` is the one window function eventstats supports), so raising those needs queries nobody has written yet. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/README.md | 23 ++++ .../__tests__/harvest-queries.test.mjs | 101 ++++++++++++++++ scripts/ppl-lint/harvest-queries.mjs | 111 +++++++++++++++++- 3 files changed, 231 insertions(+), 4 deletions(-) diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index a7cbd31714b..f4eb2bb89e2 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -432,6 +432,20 @@ harvest-queries.mjs ──▶ discovery-corpus.json ──┬──▶ run-front are unescaped so the query matches what the test actually linted. Against OSD `main` today this yields **~109 queries across 12 rules** versus 27 across 11 in the enforced corpus. + + Each file's **lint context** is harvested alongside its queries. Seven of the + nineteen rules are `needsContext: true` and self-suppress without a `typeMap`, so + harvesting queries alone produced 26 `rex-scan-cost` queries and zero triggers — + the detector never ran, which in the report is indistinguishable from a rule that + fired on nothing. The context is taken from the test file (its `typeMap`, + `disabledObjectFields`) because its author wrote it to make exactly those queries + fire; a hand-written substitute would be a guess about which field types each + query depends on, and a wrong guess silently suppresses the detector again. A rule + tested under two different contexts gets two spec files rather than a merged one. + + A **wildcard** source is deliberately not remapped: `wildcard-source-zero-match` + exists to flag a pattern matching no visible index, so rewriting `source=\`nope-*\`` + to a concrete index destroys the only thing it detects. 2. **Observe both halves.** `--specs-out` writes the corpus as ordinary spec files so the **existing** detector runner produces real diagnostic counts with no changes to it; a non-zero exit is expected there and ignored, because the generated @@ -475,6 +489,15 @@ The report also prints per-rule trigger counts and whether each rule has enough question above: a rule showing **1 trigger** cannot distinguish the two, and a rule showing **0** was not observed at all. +Against OSD `main` on the compiled surface this currently yields **41 triggers with +9 of 12 rules at ≥2**. The three that remain at one trigger are at the ceiling of +what OSD's tests contain — `agg-on-text`, `wildcard-source-zero-match` and +`unsupported-window-function-in-eventstats` each have exactly one trigger written +there, and their other queries are genuine controls (`stats avg(balance)` on a +numeric field is valid; `row_number` is the one window function eventstats +supports). Raising those needs queries nobody has written yet — the point where +generation, rather than harvesting, is what adds coverage. + This job is `continue-on-error: true` and the labeler always exits zero. A finding here is a lead, not a proven defect; failing unrelated PRs on an auto-generated guess would destroy the check's credibility. It runs against one engine (the newest diff --git a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs index fc9bf7d8276..42539f4529e 100644 --- a/scripts/ppl-lint/__tests__/harvest-queries.test.mjs +++ b/scripts/ppl-lint/__tests__/harvest-queries.test.mjs @@ -20,6 +20,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { + harvestContext, harvestFile, referencedIdentifiers, remapIndex, @@ -162,6 +163,106 @@ test('remapping is a no-op without a target index', () => { assert.equal(remapIndex('source=logs | fields a', ''), 'source=logs | fields a'); }); +test('a WILDCARD source is left alone', () => { + // Found by running the pipeline: rewriting `source=`nope-*`` to a concrete index + // destroyed the only thing `wildcard-source-zero-match` detects, so its single + // harvested query became a control and the rule reported zero triggers. + assert.equal(remapIndex('source=`nope-*`', 'acct'), 'source=`nope-*`'); + assert.equal(remapIndex('source=logs-* | fields a', 'acct'), 'source=logs-* | fields a'); + assert.equal(remapIndex('index=a* | head 1', 'acct'), 'index=a* | head 1'); +}); + +test('a non-wildcard source is still remapped when a wildcard appears elsewhere', () => { + // The guard must key on a wildcard in the SOURCE, not anywhere in the query — a + // regex or a field list containing `*` is unrelated to index resolution. + assert.equal( + remapIndex('source=logs | rex field=m "(?.*)"', 'acct'), + 'source=acct | rex field=m "(?.*)"' + ); +}); + +// --- context harvesting ------------------------------------------------------ + +test('a typeMap declaration is harvested', () => { + // Seven of nineteen rules are needsContext and self-suppress without this. The + // context is taken from the test file because its author wrote it to make exactly + // these queries fire; a hand-written substitute would be a guess. + const source = ` + const typeMap = new Map([ + ['age', 'long'], + ['firstname', 'text'], + ['attributes', 'flat_object'], + ]); + `; + assert.deepEqual(harvestContext(source).typeMap, { + age: 'long', + firstname: 'text', + attributes: 'flat_object', + }); +}); + +test('disabledObjectFields is harvested', () => { + const source = "const ctx = { typeMap, disabledObjectFields: new Set(['raw', 'blob']) };"; + assert.deepEqual(harvestContext(source).disabledObjectFields, ['raw', 'blob']); +}); + +test('a file with no context declaration yields an empty context', () => { + const out = harvestContext("describe('x', () => { lint('source=a | head 1'); });"); + assert.deepEqual(out.typeMap, {}); + assert.deepEqual(out.disabledObjectFields, []); +}); + +test('generated specs carry the harvested context as frontendContext', () => { + const corpus = { + index: 'acct', + queries: [ + { + ruleId: 'flat-object-subfield', + name: 'd0', + query: 'source=acct | where attributes.x = 1', + context: { typeMap: { attributes: 'flat_object' }, disabledObjectFields: ['raw'] }, + }, + ], + }; + const spec = toRunnerSpecs(corpus)[0].spec; + assert.deepEqual(spec.frontendContext.deriveFromMapping, { attributes: 'flat_object' }); + assert.deepEqual(spec.frontendContext.disabledObjectFields, ['raw']); + // Two rules ship `enabled: false` and only run when the host overrides them. + assert.equal(spec.frontendContext.forceEnable, true); +}); + +test('visibleIndices is supplied even with no typeMap', () => { + // `wildcard-source-zero-match` reads ONLY visibleIndices and self-suppresses on an + // empty list; its test file declares no typeMap, so keying this off the mapping + // left the rule permanently inert. + const spec = toRunnerSpecs({ + index: 'acct', + queries: [ + { ruleId: 'wildcard-source-zero-match', name: 'd0', query: 'source=`nope-*`', context: {} }, + ], + })[0].spec; + assert.deepEqual(spec.frontendContext.visibleIndices, ['{{index}}']); +}); + +test('one rule tested under two different contexts yields two specs', () => { + // Merging them would hand a query field types its own test never used, so the + // verdict would describe a scenario nobody wrote. + const corpus = { + index: 'acct', + queries: [ + { ruleId: 'rex-scan-cost', name: 'd0', query: 'source=acct | rex field=a ""', context: { typeMap: { a: 'text' } } }, + { ruleId: 'rex-scan-cost', name: 'd1', query: 'source=acct | rex field=b ""', context: { typeMap: { b: 'keyword' } } }, + ], + }; + const specs = toRunnerSpecs(corpus); + assert.equal(specs.length, 2); + // Suffixed only when a rule actually has more than one context. + assert.deepEqual(specs.map((s) => s.fileName).sort(), [ + 'rex-scan-cost.1.discovery.spec.json', + 'rex-scan-cost.2.discovery.spec.json', + ]); +}); + test('the original query is kept alongside the remapped one', () => { // Needed to explain a finding: a reader has to be able to see what the OSD test // actually asserted before trusting a disagreement derived from the rewrite. diff --git a/scripts/ppl-lint/harvest-queries.mjs b/scripts/ppl-lint/harvest-queries.mjs index b641f4612b1..82023a8b187 100644 --- a/scripts/ppl-lint/harvest-queries.mjs +++ b/scripts/ppl-lint/harvest-queries.mjs @@ -286,6 +286,15 @@ function unescapeJsString(raw) { */ export function remapIndex(query, targetIndex) { if (!targetIndex) return query; + // A WILDCARD source is left alone. `wildcard-source-zero-match` exists precisely + // to flag a pattern matching no visible index, so rewriting `source=\`nope-*\`` + // to a concrete index destroys the only thing the rule detects — the query became + // a control and the rule reported zero triggers. More generally, a wildcard is + // part of the query's meaning rather than an incidental index name, and the + // engine resolves a non-matching pattern on its own without erroring. + if (/\bsource\s*=\s*`?[^\s`|,]*\*/.test(query) || /\bindex\s*=\s*`?[^\s`|,]*\*/.test(query)) { + return query; + } let out = query .replace(/\bsource\s*=\s*`[^`]+`/g, `source=${targetIndex}`) .replace(/\bsource\s*=\s*[A-Za-z_][\w.*-]*/g, `source=${targetIndex}`) @@ -348,8 +357,54 @@ export function ruleFromDescribeTitle(title, knownRules) { return best; } +/** + * Harvest the lint CONTEXT a test file declares, not just its queries. + * + * Seven of nineteen rules are `needsContext: true` — they self-suppress without a + * `typeMap`, and `enabled-false-object` additionally needs `disabledObjectFields`. + * Harvesting their queries without their context produced 26 `rex-scan-cost` + * queries and zero triggers: the detector never ran, which is indistinguishable in + * the report from a rule that fired on nothing. + * + * The context is taken from the file rather than invented because the OSD test + * author wrote it to make exactly these queries fire. A hand-written substitute + * would be a guess about which field types each query depends on, and a wrong guess + * silently suppresses the detector again. + * + * Parses the conventional shapes those files use: + * const typeMap = new Map([ ['age', 'long'], ... ]); + * disabledObjectFields: new Set(['raw']), + * + * Regex rather than a TS parser for the same reason as `buildDescribeScopes`: these + * are conventional declarations, and the failure mode is an empty context, which + * leaves the rule visibly at zero triggers rather than producing a wrong verdict. + */ +export function harvestContext(source) { + const typeMap = {}; + // Every `['name', 'type']` pair inside a `new Map...([ ... ])` initializer. Scoped + // to Map literals so unrelated tuple arrays in the file are not picked up. + for (const mapMatch of source.matchAll(/new Map\s*(?:<[^>]*>)?\s*\(\s*\[([\s\S]*?)\]\s*\)/g)) { + for (const pair of mapMatch[1].matchAll(/\[\s*'([^']+)'\s*,\s*'([^']+)'\s*\]/g)) { + typeMap[pair[1]] = pair[2]; + } + } + + const disabledObjectFields = []; + for (const match of source.matchAll(/disabledObjectFields:\s*new Set\s*\(\s*\[([^\]]*)\]/g)) { + for (const item of match[1].matchAll(/'([^']+)'/g)) { + disabledObjectFields.push(item[1]); + } + } + + return { + typeMap, + disabledObjectFields: [...new Set(disabledObjectFields)], + }; +} + /** Harvest one file into `{ ruleId, query, source }` records. */ export function harvestFile(source, { file, knownRules, index }) { + const context = harvestContext(source); const scopes = buildDescribeScopes(source); const known = new Set(knownRules || []); const out = []; @@ -384,6 +439,10 @@ export function harvestFile(source, { file, knownRules, index }) { originalQuery: query, identifiers: referencedIdentifiers(query), source: `${file}:${line}`, + // Carried per query, not per rule: two files can test the same rule with + // different field types, and merging them would give a query a typeMap its + // own test never used. + context, }); } return out; @@ -410,15 +469,34 @@ export function harvestFile(source, { file, knownRules, index }) { * One spec per rule, because the runner keys wiring checks off `spec.ruleId`. */ export function toRunnerSpecs(corpus) { + // Grouped by rule AND by harvested context. A `needsContext` rule self-suppresses + // without a typeMap, so a query has to be scored under the context its own test + // declared — merging two files' contexts into one spec would hand a query field + // types its test never used, and the resulting verdict would describe a scenario + // nobody wrote. const byRule = new Map(); for (const [i, entry] of (corpus.queries || []).entries()) { if (!entry.ruleId) continue; - if (!byRule.has(entry.ruleId)) byRule.set(entry.ruleId, []); - byRule.get(entry.ruleId).push({ ...entry, name: entry.name || `discovery-${i}` }); + const contextKey = JSON.stringify(entry.context || {}); + const key = `${entry.ruleId}${contextKey}`; + if (!byRule.has(key)) { + byRule.set(key, { ruleId: entry.ruleId, context: entry.context, entries: [] }); + } + byRule.get(key).entries.push({ ...entry, name: entry.name || `discovery-${i}` }); } + // A rule with more than one distinct context needs more than one spec file, so + // names are suffixed only when that happens — keeping the common case readable. + const groupCount = new Map(); + for (const { ruleId } of byRule.values()) { + groupCount.set(ruleId, (groupCount.get(ruleId) || 0) + 1); + } + const seenPerRule = new Map(); + const specs = []; - for (const [ruleId, entries] of [...byRule].sort()) { + for (const [, group] of [...byRule].sort((a, b) => a[0].localeCompare(b[0]))) { + const { ruleId, context } = group; + const entries = group.entries; const queries = {}; const expected = {}; for (const entry of entries) { @@ -429,8 +507,32 @@ export function toRunnerSpecs(corpus) { queries[entry.name] = { role: 'trigger', query: entry.query }; expected[entry.name] = { detectorCount: 0 }; } + const ordinal = (seenPerRule.get(ruleId) || 0) + 1; + seenPerRule.set(ruleId, ordinal); + const suffix = groupCount.get(ruleId) > 1 ? `.${ordinal}` : ''; + + const typeMap = (context && context.typeMap) || {}; + const disabledObjectFields = (context && context.disabledObjectFields) || []; + const frontendContext = { isCalcite: true }; + if (Object.keys(typeMap).length > 0) { + // `deriveFromMapping` is what the runner turns into `fields` + `typeMap`, the + // context every `needsContext` rule requires before it will emit anything. + frontendContext.deriveFromMapping = typeMap; + } + // Always supplied, independent of the typeMap. `wildcard-source-zero-match` + // reads ONLY `visibleIndices` and self-suppresses on an empty list (otherwise + // every wildcard would false-fire "matched 0 of 0") — and its test file declares + // no typeMap, so keying this off the mapping left the rule permanently inert. + frontendContext.visibleIndices = ['{{index}}']; + if (disabledObjectFields.length > 0) { + frontendContext.disabledObjectFields = disabledObjectFields; + } + // Two rules ship `enabled: false` and only run when the host overrides them. + // Without this they are inert and every harvested query reads as a control. + frontendContext.forceEnable = true; + specs.push({ - fileName: `${ruleId}.discovery.spec.json`, + fileName: `${ruleId}${suffix}.discovery.spec.json`, spec: { schemaVersion: 3, ruleId, @@ -440,6 +542,7 @@ export function toRunnerSpecs(corpus) { // present, and a mismatch there would fail the run for a reason that has // nothing to do with discovery. index: corpus.index || undefined, + frontendContext, queries, // A single open expectation so exactly one entry matches every engine // version; the pinned counts are placeholders (see the note above). From f35ad5cf8be5689e92971927b43c93f2b20e1d25 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:39:40 -0700 Subject: [PATCH 21/39] fix(test): write the target manifest even without a grammar bundle The 3.5.0 compiled-surface leg observed all 27 contract cases against the real engine and reported success, but uploaded no target.json -- and the multi-version aggregator treats a leg without one as fatal, so those observations were unusable. Cause: exportGrammarArtifacts returns early when -Dppl.lint.grammar.bundle is unset, and the target manifest was written inside that same block. A compiled-surface leg deliberately omits the bundle flag (its engine predates the grammar endpoint), so it silently produced a leg with no engine version recorded. Write the manifest on both paths. Every consumer keys on engineVersion, which has nothing to do with whether a bundle exists; grammarHash and grammarBundle are simply empty when there is none. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 0a0b686cc44..e9d1ad6a947 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -606,6 +606,13 @@ JSONObject toJson() { private void exportGrammarArtifacts(List failures) { String bundlePath = System.getProperty("ppl.lint.grammar.bundle"); if (bundlePath == null || bundlePath.isEmpty()) { + // No bundle requested. That is a compiled-surface leg (an engine predating + // GET /_plugins/_ppl/_grammar) or a local run. The target manifest still has + // to be written: it carries the engine version every consumer keys on, and + // the multi-version aggregator treats a leg without one as fatal. Writing it + // only alongside the bundle silently produced legs the aggregator could not + // read. + writeTargetManifest("", failures); return; } try { @@ -615,16 +622,7 @@ private void exportGrammarArtifacts(List failures) { JSONObject bundle = new JSONObject(bundleBody); String grammarHash = bundle.optString("grammarHash", ""); - - String targetPath = System.getProperty("ppl.lint.target"); - if (targetPath != null && !targetPath.isEmpty()) { - JSONObject target = - new JSONObject() - .put("engineVersion", engineVersionRaw == null ? "" : engineVersionRaw) - .put("grammarHash", grammarHash) - .put("grammarBundle", Paths.get(bundlePath).getFileName().toString()); - Files.write(Paths.get(targetPath), target.toString(2).getBytes(StandardCharsets.UTF_8)); - } + writeTargetManifest(grammarHash, Paths.get(bundlePath).getFileName().toString(), failures); log("_grammar", "export", "wrote candidate bundle (" + grammarHash + ") to " + bundlePath); } catch (Exception e) { failures.add( @@ -632,6 +630,33 @@ private void exportGrammarArtifacts(List failures) { } } + /** Target manifest for a leg with no grammar bundle (compiled surface / local run). */ + private void writeTargetManifest(String grammarHash, List failures) { + writeTargetManifest(grammarHash, "", failures); + } + + /** + * Write {@code ppl.lint.target}: the engine version, the grammar hash when there is one, and the + * bundle filename when one was exported. Every consumer keys on {@code engineVersion}, so this is + * written whether or not a bundle exists. + */ + private void writeTargetManifest(String grammarHash, String bundleName, List failures) { + String targetPath = System.getProperty("ppl.lint.target"); + if (targetPath == null || targetPath.isEmpty()) { + return; + } + try { + JSONObject target = + new JSONObject() + .put("engineVersion", engineVersionRaw == null ? "" : engineVersionRaw) + .put("grammarHash", grammarHash) + .put("grammarBundle", bundleName); + Files.write(Paths.get(targetPath), target.toString(2).getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + failures.add("[grammar-export] failed to write " + targetPath + ": " + e.getMessage()); + } + } + // --- cluster settings ------------------------------------------------------ /** True when the contract's fixture leaves Calcite enabled (the default). */ From 000d4feca37ea8f0260e70664d511db8fefb38d4 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:40:41 -0700 Subject: [PATCH 22/39] fix(ci): run discovery on the runtime-bundle surface, not only the compiled one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The discovery job hardcoded `PPL_LINT_SURFACE=compiled-simplified`. That was an unnecessary restriction: `lint_runner` SKIPS the four `runtimeOnly` rules on the compiled grammar because the productions they walk do not exist there, so a compiled-only run cannot observe them at all — and three of the four ship at error severity, where a false positive is most expensive. The job now exports the engine's grammar via GET /_plugins/_ppl/_grammar and lints on the runtime surface, falling back to the compiled surface with a warning if the export fails. Best-effort rather than fatal: a lead-generator that produces nothing because one endpoint was unavailable is worse than one with narrower coverage, and the surface is recorded in the report so a reader can tell which ran. Also fixes an unbound-variable crash in that step. Expanding an empty array as "${extra[@]}" under `set -u` is an error in bash before 4.4, so the compiled-surface fallback would have died — the one path that only runs when something else already went wrong. Verified both branches. Worth recording since it bounds what harvesting can achieve: the four runtimeOnly rules are at zero harvested queries and no surface changes that. OSD's lint tests contain no trigger for union-min-datasets, multisearch-min-subsearch or replace-wildcard-asymmetry; the only place they appear is a negative assertion that they no-op on the compiled surface. Harvesting cannot invent what was never written. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 46 ++++++++++++++++++- scripts/ppl-lint/README.md | 15 +++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 2438e665fbe..1a8aefbb535 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -830,14 +830,58 @@ jobs: -H 'content-type: application/json' \ -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' + # Export this engine's grammar bundle so the detector pass can run on the + # RUNTIME surface. That surface matters more than the compiled one here: the + # four `runtimeOnly` rules (union/multisearch/replace arity) are SKIPPED by + # lint_runner on the compiled grammar because the productions they walk do not + # exist there — so a compiled-only discovery run cannot observe them at all, + # and three of the four ship at error severity. + - name: Export the engine grammar bundle + id: bundle + run: | + set -uo pipefail + if curl -sf --max-time 60 "http://localhost:9200/_plugins/_ppl/_grammar" \ + -o "$GITHUB_WORKSPACE/discovery-bundle.json"; then + hash=$(python3 -c " + import json + print(json.load(open('$GITHUB_WORKSPACE/discovery-bundle.json')).get('grammarHash','')) + ") + python3 -c " + import json + json.dump({'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '$hash'}, + open('$GITHUB_WORKSPACE/discovery-target.json','w')) + " + echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" + else + # Not fatal. Discovery is best-effort, and the compiled surface still + # covers 12 of the rules — a lead-generator that produces nothing because + # one endpoint was unavailable is worse than one with narrower coverage. + # The surface is recorded in the report, so a reader can see which ran. + echo "::warning::_grammar export failed; falling back to the compiled surface (runtimeOnly rules will not be observed)." + echo "surface=compiled-simplified" >> "$GITHUB_OUTPUT" + fi + - name: Run the detectors over the discovery corpus working-directory: .ci/OpenSearch-Dashboards + env: + SURFACE: ${{ steps.bundle.outputs.surface }} run: | set -uo pipefail + # Seeded with a harmless assignment rather than left empty: under `set -u`, + # expanding an empty array as "${a[@]}" is an unbound-variable error in bash + # before 4.4, which would crash the compiled-surface fallback — the very + # path that only runs when something else already went wrong. + extra=(PPL_LINT_DISCOVERY=1) + if [ "$SURFACE" = 'runtime-bundle' ]; then + extra+=(PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" + PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") + fi # A non-zero exit is EXPECTED and ignored: the generated specs carry # placeholder expectations, so the runner reports a "failure" for every # query whose real diagnostic count differs. Only the report is read. - PPL_LINT_SURFACE=compiled-simplified \ + env "${extra[@]}" \ + PPL_LINT_SURFACE="$SURFACE" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ PPL_LINT_SCHEDULE=nightly \ PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index f4eb2bb89e2..e1e6a1439d6 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -489,7 +489,7 @@ The report also prints per-rule trigger counts and whether each rule has enough question above: a rule showing **1 trigger** cannot distinguish the two, and a rule showing **0** was not observed at all. -Against OSD `main` on the compiled surface this currently yields **41 triggers with +Measured on the compiled surface against OSD `main`, this yields **41 triggers with 9 of 12 rules at ≥2**. The three that remain at one trigger are at the ceiling of what OSD's tests contain — `agg-on-text`, `wildcard-source-zero-match` and `unsupported-window-function-in-eventstats` each have exactly one trigger written @@ -498,6 +498,19 @@ numeric field is valid; `row_number` is the one window function eventstats supports). Raising those needs queries nobody has written yet — the point where generation, rather than harvesting, is what adds coverage. +The job prefers the **runtime-bundle** surface, exporting the engine's grammar via +`GET /_plugins/_ppl/_grammar` and falling back to the compiled surface (with a +warning) if that fails. The runtime surface matters because `lint_runner` SKIPS the +four `runtimeOnly` rules on the compiled grammar — the productions they walk do not +exist there — and three of those ship at error severity. + +Those four are nonetheless still at **zero** harvested queries, and no surface fixes +that: OSD's lint tests contain no trigger for `union-min-datasets`, +`multisearch-min-subsearch` or `replace-wildcard-asymmetry` at all. The only place +they appear is a negative assertion that they no-op on the compiled surface +(`analyzer_lint.test.ts`, "runtime-only rules no-op"). Harvesting cannot invent what +was never written, so these are generation's job, not the harvester's. + This job is `continue-on-error: true` and the labeler always exits zero. A finding here is a lead, not a proven defect; failing unrelated PRs on an auto-generated guess would destroy the check's credibility. It runs against one engine (the newest From a19bdf689fd67d7be7186e670a13329468993f63 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 09:53:25 -0700 Subject: [PATCH 23/39] test(ppl-lint): add a second trigger to every enforced contract that had one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The partial-vs-full relaxation verdict reads the ENFORCED corpus, and 8 of its 11 contracts pinned exactly one trigger. With one trigger, "every trigger relaxed" is a single observation, so the verdict cannot distinguish a full engine fix (version-scope the rule away) from a partial one (narrow the detector) — and those need opposite actions. The classifier warns about it, but the fix is more triggers. Each new trigger exercises a DIFFERENT shape of the same condition, so a partial engine fix is visible as a disagreement between them rather than as a uniform flip: union-min-datasets single dataset that carries a pipeline (| fields) multisearch-min-subsearch single subsearch that carries a pipeline (| where) replace-wildcard-asymmetry reversed asymmetry (2 wildcards -> 1, not 1 -> 2) invalid-capture-group-name hyphen, not just underscore unsupported-window-function dense_rank, not just rank dedup-consecutive-unsupported multi-field dedup with consecutive=true division-by-zero decimal 0.0, not just integer 0 head-without-sort head after a where stage, not a bare source Every expectation was verified on a live 3.8 engine rather than inferred, including the exact error type and reason string: union/fields 400 IllegalArgumentException Union command requires ... Provided: 1 multisearch/where 400 SyntaxCheckException Invalid Query replace 2->1 400 IllegalArgumentException pattern has 2 wildcard(s), replacement has 1 rex hyphen 400 IllegalArgumentException Invalid capture group name 'user-name'. eventstats dense 400 CalciteUnsupportedException Unexpected window function: dense_rank dedup multi-field 200 (advisory; succeeds via the Calcite-to-v2 fallback) head after where 200 (advisory) balance / 0.0 200 with the ratio column all-null Detector counts and severities were confirmed by running the real detector runner over the corpus: all four compiled-surface triggers score 1 at the contracted severity, and eventstats-dense-rank scores 1/error once a version is supplied. The four runtime-bundle-only contracts are reported not-applicable on the compiled surface, as before. Failure count is unchanged from baseline (8, all pre-existing "enabled catalog rule has no contract file" coverage warnings). Worth noting for review: the pre-3.8 expectations for eventstats-dense-rank reuse the existing rank() pins (500 / UnsupportedOperationException) by analogy — both are CalciteUnsupportedException on 3.8, and only 3.8 was available to verify. The 3.6 and 3.7 legs will confirm or correct them. Signed-off-by: Hanyu Wei --- .../dedup-consecutive-unsupported.spec.json | 45 ++++++++- .../contracts/division-by-zero.spec.json | 50 ++++++++-- .../contracts/head-without-sort.spec.json | 40 +++++++- .../invalid-capture-group-name.spec.json | 34 +++++++ .../multisearch-min-subsearch.spec.json | 55 +++++++++-- .../replace-wildcard-asymmetry.spec.json | 34 +++++++ .../contracts/union-min-datasets.spec.json | 57 ++++++++++-- ...ed-window-function-in-eventstats.spec.json | 91 +++++++++++++++++-- 8 files changed, 370 insertions(+), 36 deletions(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index 90a614307b7..fb9d44de7c3 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -10,11 +10,19 @@ "runtimeOnly": false, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.3.0", "engine": "calcite" } + "appliesTo": { + "minVersion": "3.3.0", + "engine": "calcite" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": true } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": true + } }, "frontendContext": { "isCalcite": true @@ -25,6 +33,10 @@ "role": "trigger", "query": "source={{index}} | dedup firstname consecutive=true" }, + "dedup-consecutive-true-multi-field": { + "role": "trigger", + "query": "source={{index}} | dedup firstname, lastname consecutive=true" + }, "dedup-plain-control": { "role": "control", "query": "source={{index}} | dedup firstname" @@ -38,11 +50,34 @@ "dedup-consecutive-true": { "detectorCount": 1, "severity": "warning", - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "dedup-consecutive-true-multi-field": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } }, "dedup-plain-control": { "detectorCount": 0, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index 5e1b33884c8..0303e32e2a5 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "division-by-zero", - "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL — it documents that boundary rather than asserting a gap.", + "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL \u2014 it documents that boundary rather than asserting a gap.", "grammarSurface": "both", "schedule": "nightly", "wiring": { @@ -14,8 +14,13 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -26,6 +31,10 @@ "role": "trigger", "query": "source={{index}} | eval ratio = balance / 0 | fields ratio | head 1" }, + "divide-by-decimal-zero-literal": { + "role": "trigger", + "query": "source={{index}} | eval ratio = balance / 0.0 | fields ratio | head 1" + }, "divide-by-nonzero-control": { "role": "control", "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" @@ -42,15 +51,44 @@ "divide-by-zero-literal": { "detectorCount": 1, "severity": "warning", - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "ratio" } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + } + }, + "divide-by-decimal-zero-literal": { + "detectorCount": 1, + "severity": "warning", + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "ratio" + } + } }, "divide-by-nonzero-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } }, "modulo-by-zero-not-flagged": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "columnAllNull": "m" } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "columnAllNull": "m" + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index 695f1b6b550..68cd387a88c 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -13,8 +13,13 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -25,6 +30,10 @@ "role": "trigger", "query": "source={{index}} | head 5" }, + "head-without-sort-after-where": { + "role": "trigger", + "query": "source={{index}} | where age > 20 | head 5" + }, "head-with-sort-control": { "role": "control", "query": "source={{index}} | sort age | head 5" @@ -37,11 +46,34 @@ "head-without-sort": { "detectorCount": 1, "severity": "info", - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } + }, + "head-without-sort-after-where": { + "detectorCount": 1, + "severity": "info", + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } }, "head-with-sort-control": { "detectorCount": 0, - "backend": { "kind": "advisory", "httpStatus": 200, "expect": { "accepted": true } } + "backend": { + "kind": "advisory", + "httpStatus": 200, + "expect": { + "accepted": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json index 3d39fa2e3dc..d96142d62e0 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/invalid-capture-group-name.spec.json @@ -37,6 +37,10 @@ "role": "trigger", "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" }, + "rex-capture-name-hyphen": { + "role": "trigger", + "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email" + }, "rex-capture-name-alphanumeric-control": { "role": "control", "query": "source={{index}} | rex field=email \"(?[^@]+)@(?.+)\" | fields email, username, domain | head 1" @@ -62,6 +66,21 @@ } } }, + "rex-capture-name-hyphen": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, "backend": { @@ -93,6 +112,21 @@ } } }, + "rex-capture-name-hyphen": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid capture group name 'user-name'." + } + } + } + }, "rex-capture-name-alphanumeric-control": { "detectorCount": 0, "backend": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json index 018d086fec3..345742aa9d8 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/multisearch-min-subsearch.spec.json @@ -3,8 +3,11 @@ "ruleId": "multisearch-min-subsearch", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["multisearchCommand", "subSearch"], - "notes": "Query-initial (no leading pipe) on purpose — see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", + "requiredParserRules": [ + "multisearchCommand", + "subSearch" + ], + "notes": "Query-initial (no leading pipe) on purpose \u2014 see the note on union-min-datasets. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, which would change the effective parse relative to what the backend receives. A query-initial 'multisearch [...]' is sent byte-identically to both halves.", "wiring": { "detector": "multisearch-min-subsearch", "enabled": true, @@ -12,11 +15,18 @@ "runtimeOnly": true, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.4.0" } + "appliesTo": { + "minVersion": "3.4.0" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -27,6 +37,10 @@ "role": "trigger", "query": "multisearch [ search source={{index}} ]" }, + "multisearch-single-subsearch-with-where": { + "role": "trigger", + "query": "multisearch [ search source={{index}} | where age > 30 ]" + }, "multisearch-two-subsearches-control": { "role": "control", "query": "multisearch [ search source={{index}} ] [ search source={{index}} ]" @@ -42,12 +56,39 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SyntaxCheckException", "reason": "Invalid Query" } } + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } + } + }, + "multisearch-single-subsearch-with-where": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "SyntaxCheckException", + "reason": "Invalid Query" + } + } } }, "multisearch-two-subsearches-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json index 593bb92023d..c6fa5b162b5 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/replace-wildcard-asymmetry.spec.json @@ -37,6 +37,10 @@ "role": "trigger", "query": "source={{index}} | replace \"*_a\" with \"b_*_*\" in firstname" }, + "replace-wildcard-count-mismatch-reverse": { + "role": "trigger", + "query": "source={{index}} | replace \"*_a_*\" with \"b_*\" in firstname" + }, "replace-symmetric-control": { "role": "control", "query": "source={{index}} | replace \"*_a\" with \"b_*\" in firstname | head 1" @@ -62,6 +66,21 @@ } } }, + "replace-wildcard-count-mismatch-reverse": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } + } + } + }, "replace-symmetric-control": { "detectorCount": 0, "backend": { @@ -93,6 +112,21 @@ } } }, + "replace-wildcard-count-mismatch-reverse": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Error in 'replace' command: Wildcard count mismatch - pattern has 2 wildcard(s), replacement has 1. Replacement must have same number of wildcards or none." + } + } + } + }, "replace-symmetric-control": { "detectorCount": 0, "backend": { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json index 7f110cc9423..d5eadcc25f5 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/union-min-datasets.spec.json @@ -3,8 +3,12 @@ "ruleId": "union-min-datasets", "grammarSurface": "runtime-bundle", "schedule": "pr", - "requiredParserRules": ["unionCommand", "unionDataset", "pplCommands"], - "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' — a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", + "requiredParserRules": [ + "unionCommand", + "unionDataset", + "pplCommands" + ], + "notes": "Query-initial (no leading pipe) on purpose. OSD's runtime lint prepends a synthetic 'source=t ' prefix to pipe-first queries, so linting '| union [...]' actually parses 'source=t | union [...]' \u2014 a valid MID-pipeline union (implicit upstream dataset) that the detector deliberately does not flag. The backend, receiving the raw pipe-first query, would still reject it, so a pipe-first trigger makes the two halves test different effective queries (violating the design's 'same queries' rule). A query-initial 'union [...]' is sent byte-identically to both sides and keeps the differential sound.", "wiring": { "detector": "union-min-datasets", "enabled": true, @@ -12,11 +16,19 @@ "runtimeOnly": true, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.7.0", "engine": "calcite" } + "appliesTo": { + "minVersion": "3.7.0", + "engine": "calcite" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -27,6 +39,10 @@ "role": "trigger", "query": "union [ source={{index}} ]" }, + "union-single-dataset-with-fields": { + "role": "trigger", + "query": "union [ source={{index}} | fields firstname ]" + }, "union-two-datasets-control": { "role": "control", "query": "union [ source={{index}} ] [ source={{index}} ]" @@ -43,12 +59,39 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "IllegalArgumentException", "reason": "Union command requires at least two datasets. Provided: 1" } } + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + } + }, + "union-single-dataset-with-fields": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } } }, "union-two-datasets-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 4f436a383cb..6479fd2b1b1 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -11,11 +11,18 @@ "runtimeOnly": false, "needsContext": false, "needsExplain": false, - "appliesTo": { "minVersion": "3.4.0" } + "appliesTo": { + "minVersion": "3.4.0" + } }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } }, "frontendContext": { "isCalcite": true @@ -26,6 +33,10 @@ "role": "trigger", "query": "source={{index}} | eventstats rank() as rank_value" }, + "eventstats-dense-rank": { + "role": "trigger", + "query": "source={{index}} | eventstats dense_rank() as rank_value" + }, "eventstats-avg-control": { "role": "control", "query": "source={{index}} | eventstats avg(age) as avg_age" @@ -51,9 +62,30 @@ } } }, + "eventstats-dense-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } }, @@ -75,9 +107,30 @@ } } }, + "eventstats-dense-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 500, + "body": { + "status": 500, + "error": { + "type": "UnsupportedOperationException", + "reason": "There was internal problem at backend" + } + } + } + }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } }, @@ -92,13 +145,37 @@ "httpStatus": 400, "body": { "status": 400, - "error": { "type": "CalciteUnsupportedException", "reason": "Unexpected window function: rank" } + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: rank" + } + } + } + }, + "eventstats-dense-rank": { + "detectorCount": 1, + "severity": "error", + "backend": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "CalciteUnsupportedException", + "reason": "Unexpected window function: dense_rank" + } } } }, "eventstats-avg-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } From 98c6d12b794e0333a410189234597dbd532355e9 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 10:00:14 -0700 Subject: [PATCH 24/39] test(ci): probe REST client connectivity from a JVM Seven hypotheses for the 2.19 observation-leg timeout have each been refuted by observation: the index-wipe race, a Gradle cluster fallback, HTTP/2 negotiation, FIPS, the bundled plugin set, a port collision, and address family. What is established is narrow and contradictory: the engine is alive and logging throughout, its publish address and network topology are identical to the passing 3.5.0 leg, the Gradle args and task graphs are byte-identical, curl reaches every endpoint the framework calls in 0s from the same runner -- and GET _nodes/plugins from the test JVM never returns. curl has said everything it can. This probe asks the JVM instead, one layer at a time against the same address: raw TCP connect, then HttpURLConnection, then the real OpenSearch RestClient on each endpoint the framework itself calls, each timed and bounded at 15s. Whichever layer stops working localizes the fault -- network, JDK HTTP stack, async client, or a specific response. It deliberately does not extend the framework's base class, since that base class is what hangs; inheriting it would reproduce the symptom instead of isolating it. Reports rather than asserts (the leg is already failing and the evidence is the point), except that a failed TCP connect is fatal because nothing below it would mean anything. Wired into the compiled leg with continue-on-error so a diagnostic can never be what decides the leg. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 32 ++- .../remote/RestClientConnectivityProbeIT.java | 207 ++++++++++++++++++ 2 files changed, 233 insertions(+), 6 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 1a8aefbb535..19eae573de0 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -393,12 +393,6 @@ jobs: echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" fi done - # The requests above all succeed over HTTP/1.1 (curl's default) yet the - # test client still times out on the same endpoint. The remaining - # difference is protocol negotiation: RestClientBuilder builds an - # HttpAsyncClient with no version policy, which in HttpClient 5.x means - # h2-with-upgrade. Probe an explicit h2 upgrade to see whether this engine - # completes it. start=$(date +%s) if curl -sS --http2 --max-time 30 -o /dev/null -w '%{http_version}' \ "http://localhost:9200/_nodes/plugins" > /tmp/h2.out 2>/tmp/h2.err; then @@ -406,6 +400,14 @@ jobs: else echo "::warning::h2 probe FAILED after $(($(date +%s) - start))s: $(cat /tmp/h2.err)" fi + # Response SIZE is the last untested difference. curl streams the body and + # does not care; the test framework calls entityAsMap on it, and + # _nodes/plugins on an engine with many bundled plugins is large. Record the + # sizes so a size-dependent hang is visible rather than inferred. + for path in "_nodes/plugins" "_nodes" "_cat/plugins"; do + bytes=$(curl -sS --max-time 30 "http://localhost:9200/${path}" | wc -c) + echo "size probe: ${path} -> ${bytes} bytes" + done - name: Set up JDK 21 uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 @@ -413,6 +415,24 @@ jobs: distribution: 'temurin' java-version: 21 + # The curl probes above reach this engine instantly, yet the contract IT's own + # client times out on the same endpoint before any test body runs. Everything + # curl can tell us has been exhausted, so run the probe from a JVM: raw TCP, + # then HttpURLConnection, then the real OpenSearch RestClient per endpoint. + # Whichever layer stops working is the answer. + # + # `continue-on-error` because this is a diagnostic: its findings must not be + # what decides the leg, and the contract step below is still the real check. + - name: Probe REST client connectivity from a JVM + continue-on-error: true + run: | + set -uo pipefail + ./gradlew :integ-test:integTestRemote \ + --tests 'org.opensearch.sql.calcite.remote.RestClientConnectivityProbeIT' \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD' || true + - name: Run contract observation against engine ${{ matrix.version }} run: | set -euo pipefail diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java new file mode 100644 index 00000000000..0be3228ff4d --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -0,0 +1,207 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.util.Timeout; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.Response; +import org.opensearch.client.RestClient; +import org.opensearch.client.RestClientBuilder; + +/** + * Connectivity probe for {@code tests.rest.cluster}, used to diagnose why the REST test client + * cannot reach some engine versions that plain HTTP clients reach fine. + * + *

Context: on the PPL lint multi-version matrix the 2.19.0 observation leg fails with {@code + * SocketTimeoutException} after the full 60s response timeout, thrown from {@code + * OpenSearchRestTestCase.initClient} on its {@code GET _nodes/plugins} call — before any test body + * runs. From the same runner, {@code curl} against the same endpoint on the same address returns + * HTTP 200 in 0s. The 3.5.0 leg, with byte-identical Gradle args, network topology, publish address + * and task graph, passes. Seven hypotheses (index-wipe race, Gradle cluster fallback, HTTP/2 + * negotiation, FIPS, plugin set, port collision, address family) have each been refuted by + * observation. + * + *

This class deliberately does NOT extend the test framework's base class: that base class is + * what hangs, so inheriting it would reproduce the symptom without isolating the cause. Instead it + * walks up the stack one layer at a time against the same address, so a single run says exactly + * which layer stops working: + * + *

    + *
  1. raw TCP connect — is the port reachable from this JVM at all? + *
  2. {@code HttpURLConnection} — does the JDK's own HTTP stack get a response? + *
  3. {@code RestClient} with default settings — does the OpenSearch async client work? + *
  4. {@code RestClient} on the endpoints the framework itself calls, timed individually. + *
+ * + *

Every step is time-bounded and reports rather than asserts, because the point is to collect + * evidence from a leg that is already failing. The one assertion is that step 1 succeeded: if the + * JVM cannot open a socket, nothing below it means anything. + * + *

Run with: {@code ./gradlew :integ-test:integTestRemote --tests + * '*RestClientConnectivityProbeIT' -Dtests.rest.cluster=localhost:9200} + */ +public class RestClientConnectivityProbeIT { + + /** Bound well below the framework's 60s so a hang is visibly a hang, not a wait. */ + private static final Timeout PROBE_TIMEOUT = Timeout.ofSeconds(15); + + private static final String[] FRAMEWORK_ENDPOINTS = { + // The exact call OpenSearchRestTestCase.initClient makes, and the one that hangs. + "_nodes/plugins", + // What the wipe in OpenSearchSQLRestTestCase.wipeAllOpenSearchIndices calls next. + "_cat/indices?format=json&expand_wildcards=all", + // A trivial response, to separate "any request" from "this request". + "_cluster/health", + // The PPL endpoint the contract actually needs, so a pass here means the leg could work. + "_plugins/_ppl/_grammar", + }; + + @Test + public void probeConnectivity() { + String cluster = System.getProperty("tests.rest.cluster"); + if (cluster == null || cluster.isEmpty()) { + log("SKIP: -Dtests.rest.cluster not set"); + return; + } + String hostPort = cluster.split(",")[0]; + int sep = hostPort.lastIndexOf(':'); + String host = hostPort.substring(0, sep); + int port = Integer.parseInt(hostPort.substring(sep + 1)); + log("probing " + host + ":" + port); + + boolean tcpOk = probeRawSocket(host, port); + probeHttpUrlConnection(host, port); + probeRestClient(host, port); + + // Only a hard failure here is fatal: without a socket the rest is noise. + if (!tcpOk) { + throw new AssertionError("could not open a TCP connection to " + host + ":" + port); + } + } + + /** Layer 1: can this JVM open a socket to the published port? */ + private boolean probeRawSocket(String host, int port) { + long start = System.nanoTime(); + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), (int) PROBE_TIMEOUT.toMilliseconds()); + log("tcp connect OK in " + millis(start) + "ms (localAddr=" + socket.getLocalAddress() + ")"); + return true; + } catch (Exception e) { + log("tcp connect FAILED after " + millis(start) + "ms: " + describe(e)); + return false; + } + } + + /** + * Layer 2: the JDK's own blocking HTTP stack. If this works while {@code RestClient} does not, + * the problem is in the async client rather than in the network or the engine. + */ + private void probeHttpUrlConnection(String host, int port) { + long start = System.nanoTime(); + try { + URL url = new URL("http://" + host + ":" + port + "/_nodes/plugins"); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setConnectTimeout((int) PROBE_TIMEOUT.toMilliseconds()); + connection.setReadTimeout((int) PROBE_TIMEOUT.toMilliseconds()); + int status = connection.getResponseCode(); + long bytes = drain(connection); + log( + "HttpURLConnection _nodes/plugins OK in " + + millis(start) + + "ms: HTTP " + + status + + ", " + + bytes + + " bytes"); + connection.disconnect(); + } catch (Exception e) { + log("HttpURLConnection _nodes/plugins FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + + /** + * Layer 3: the real {@code RestClient}, built the way the framework builds it (defaults only, no + * credentials or TLS since these legs are plain HTTP), then each framework endpoint in turn. + * + *

Timed per endpoint: a uniform failure means the client cannot talk to this engine at all, + * while one slow endpoint among fast ones means the response itself is the problem. + */ + private void probeRestClient(String host, int port) { + RestClientBuilder builder = + RestClient.builder(new HttpHost("http", host, port)) + .setRequestConfigCallback( + config -> config.setConnectTimeout(PROBE_TIMEOUT).setResponseTimeout(PROBE_TIMEOUT)) + // The framework sets this too; without it a deprecation warning header can turn into a + // failure and confuse the diagnosis. + .setStrictDeprecationMode(false); + + try (RestClient client = builder.build()) { + for (String endpoint : FRAMEWORK_ENDPOINTS) { + long start = System.nanoTime(); + try { + Response response = client.performRequest(new Request("GET", "/" + endpoint)); + long bytes = response.getEntity() == null ? 0 : response.getEntity().getContentLength(); + log( + "RestClient " + + endpoint + + " OK in " + + millis(start) + + "ms: HTTP " + + response.getStatusLine().getStatusCode() + + ", " + + bytes + + " bytes"); + } catch (Exception e) { + log("RestClient " + endpoint + " FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + } catch (IOException e) { + log("RestClient could not be built/closed: " + describe(e)); + } + } + + private static long drain(HttpURLConnection connection) throws IOException { + byte[] buffer = new byte[8192]; + long total = 0; + try (var stream = connection.getInputStream()) { + int read; + while ((read = stream.read(buffer)) != -1) { + total += read; + } + } + return total; + } + + /** Full cause chain: the outer message alone hides which layer actually gave up. */ + private static String describe(Throwable error) { + List chain = new ArrayList<>(); + for (Throwable current = error; current != null; current = current.getCause()) { + chain.add(current.getClass().getSimpleName() + "(" + current.getMessage() + ")"); + if (current.getCause() == current) { + break; + } + } + return String.join(" <- ", chain); + } + + private static long millis(long startNanos) { + return (System.nanoTime() - startNanos) / 1_000_000; + } + + private static void log(String message) { + // stdout so it lands in the Gradle test output the CI job already prints. + System.out.println("[rest-connectivity-probe] " + message); + } +} From aa6e98156560d9d8e49328d5b170a336d88ef6ec Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 10:19:06 -0700 Subject: [PATCH 25/39] test(ppl-lint): run the whole contract corpus on every PR, not just the PR subset MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five contracts declared `schedule: "nightly"`, so a pull_request run skipped them: dedup-consecutive-unsupported, disabled-join-type, division-by-zero, field-validation and head-without-sort. That left the required check scoring 3 of 19 rows on a PR, and it hid the triggers the multi-version relaxation rollup depends on — a rule with no scored case contributes no trigger census, so full-vs-partial cannot be judged for it on a PR at all. All 11 contracts now declare `schedule: "pr"`. A PR run reaches the whole corpus: 35 queries, 19 scored on the compiled surface (the other 16 are the runtime-bundle-only contracts, not-applicable there as before), up from 3 of 19. This DOES make the four advisory rules blocking, and that is a deliberate accepted trade rather than an oversight. Neither PplLintRuleValidationIT nor run-frontend-contract.mjs consults the manifest's `enforced` list — a contract that runs is a hard assertion — so `schedule` was the only thing keeping them non-blocking. Their oracles are genuinely weaker than the error rules': an advisory rule's query SUCCEEDS, so the contract can only assert a result shape or plain acceptance, and dedup-consecutive in particular depends on the Calcite-to-v2 fallback staying enabled. If one of them goes red, check the oracle before editing a rule. The manifest description, the IT javadoc and the README all claimed the split controlled blocking. Corrected: `enforced` / `nonEnforcing` record oracle quality and review status — how much to trust a red result — not whether one can occur. The schedule filter itself is kept, since it remains the only way to hold a new contract back from PR runs while its oracle settles. Verified: the full corpus exits 0 on the PR schedule with zero skips, against a live 3.8 engine and the real ACCOUNT fixture. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 12 +++++-- .../dedup-consecutive-unsupported.spec.json | 2 +- .../contracts/disabled-join-type.spec.json | 36 +++++++++++++++---- .../contracts/division-by-zero.spec.json | 2 +- .../contracts/field-validation.spec.json | 2 +- .../contracts/head-without-sort.spec.json | 2 +- .../ppl-lint/contracts/manifest.json | 10 +++--- scripts/ppl-lint/README.md | 19 ++++++++-- 8 files changed, 66 insertions(+), 19 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index e9d1ad6a947..6a2de542ab6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -61,8 +61,16 @@ * SAME candidate grammar (design §4.2, §4.3). Export runs only when {@code * -Dppl.lint.grammar.bundle} is set (CI); local runs without it are unaffected. * - *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): PR runs only the - * fast, deterministic {@code schedule:pr} contracts; nightly runs the full corpus. + *

The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): a PR run skips + * contracts declaring {@code schedule: "nightly"}, while nightly runs the full corpus. Every + * contract in the corpus currently declares {@code schedule: "pr"}, so the two are equivalent + * today; the filter stays because it is the only mechanism for holding a new contract back from + * PR runs while its oracle is still settling. + * + *

Note that a contract which RUNS also ASSERTS. This class does not consult the manifest's + * {@code enforced} list — that list records oracle quality and review status, not blocking + * behavior. Adding a contract, or moving one onto the PR schedule, makes it capable of failing the + * required check. */ public class PplLintRuleValidationIT extends PPLIntegTestCase { diff --git a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json index fb9d44de7c3..fffdce393dd 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/dedup-consecutive-unsupported.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "dedup-consecutive-unsupported", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "dedup-consecutive-unsupported", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json index 25cd3232e88..d2d5df24260 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/disabled-join-type.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "disabled-join-type", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "disabled-join-type", "enabled": true, @@ -13,8 +13,14 @@ "appliesTo": {} }, "backendFixture": { - "indices": ["ACCOUNT"], - "clusterSettings": { "calcite": true, "calciteFallback": false, "allJoinTypesAllowed": false } + "indices": [ + "ACCOUNT" + ], + "clusterSettings": { + "calcite": true, + "calciteFallback": false, + "allJoinTypesAllowed": false + } }, "frontendContext": { "isCalcite": true @@ -44,7 +50,13 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + "body": { + "status": 400, + "error": { + "type": "SemanticCheckException", + "reason": "Invalid Query" + } + } } }, "cross-join-disabled": { @@ -53,12 +65,24 @@ "backend": { "kind": "rejection", "httpStatus": 400, - "body": { "status": 400, "error": { "type": "SemanticCheckException", "reason": "Invalid Query" } } + "body": { + "status": 400, + "error": { + "type": "SemanticCheckException", + "reason": "Invalid Query" + } + } } }, "inner-join-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backend": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } } } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index 0303e32e2a5..e1063eddc7d 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -3,7 +3,7 @@ "ruleId": "division-by-zero", "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL \u2014 it documents that boundary rather than asserting a gap.", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "division-by-zero", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json index 984fcf851c1..5814d837249 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/field-validation.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "field-validation", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "field-validation", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json index 68cd387a88c..56c272abc9c 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/head-without-sort.spec.json @@ -2,7 +2,7 @@ "schemaVersion": 3, "ruleId": "head-without-sort", "grammarSurface": "both", - "schedule": "nightly", + "schedule": "pr", "wiring": { "detector": "head-without-sort", "enabled": true, diff --git a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json index a8b031315d6..d316c1e8de1 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/manifest.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/manifest.json @@ -1,6 +1,6 @@ { "schemaVersion": 3, - "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus; `enforced` is the phase-one, reviewed, error-severity subset with a stable backend rejection oracle that blocks a PR (design §5.1, §5.2). Everything not in `enforced` runs non-blocking (nightly / advisory) until it has an equally stable oracle and owner review.", + "description": "Index of PPL lint rule validation contracts. Each entry pins one OSD analyzer rule to live SQL /_plugins/_ppl behavior. The detector runner (scripts/ppl-lint/run-frontend-contract.mjs) and backend IT (PplLintRuleValidationIT) both read these files. `contracts` is the full corpus. EVERY contract now declares `schedule: \"pr\"`, so every contract runs \u2014 and asserts \u2014 on every pull request: neither reader consults `enforced`, so any contract that runs is a hard assertion. The `enforced` / `nonEnforcing` lists below therefore describe oracle QUALITY and review status, not whether a mismatch blocks (design \u00a75.1, \u00a75.2). They are what a reviewer should read when judging how much to trust a red result.", "contracts": [ "invalid-capture-group-name.spec.json", "unsupported-window-function-in-eventstats.spec.json", @@ -38,9 +38,9 @@ "dedup-consecutive-unsupported.spec.json" ], "notes": { - "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. These block the required single-version validation-result check.", - "defaultError": "Every rule that ships enabled at ERROR severity in the OSD catalog — the set the MULTI-VERSION check enforces (scripts/ppl-lint/aggregate-versions.mjs). A default-error rule is what users cannot opt out of and what blocks a query in the editor, so it is exactly the set that must agree with every supported engine version. Kept in sync with the catalog by the coverage assertion in the aggregate step: a rules_catalog.json entry with enabled:true + severity:error and no contract file here fails the check.", - "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design §5.2) before joining `enforced`. Empty now that field-validation and flat-object-subfield are pinned across versions by the multi-version check; they remain outside single-version `enforced` because their backend oracle is a semantic 'Field [...] not found.' rejection shared with each other rather than a rule-unique grammar rejection.", - "nonEnforcing": "Warning / info / advisory / result-shape rules. They lack a stable backend rejection oracle and never block a PR; they run for coverage on the nightly schedule." + "enforced": "Reviewed error rules with a deterministic backend rejection and a valid negative control. The most trustworthy oracles in the corpus \u2014 a mismatch here is almost certainly a real drift.", + "defaultError": "Every rule that ships enabled at ERROR severity in the OSD catalog \u2014 the set the MULTI-VERSION check enforces (scripts/ppl-lint/aggregate-versions.mjs). A default-error rule is what users cannot opt out of and what blocks a query in the editor, so it is exactly the set that must agree with every supported engine version. Kept in sync with the catalog by the coverage assertion in the aggregate step: a rules_catalog.json entry with enabled:true + severity:error and no contract file here fails the check.", + "pendingReview": "Error rules awaiting Peng/Chen usefulness + false-positive review (design \u00a75.2) before joining `enforced`. Empty now that field-validation and flat-object-subfield are pinned across versions by the multi-version check; they remain outside single-version `enforced` because their backend oracle is a semantic 'Field [...] not found.' rejection shared with each other rather than a rule-unique grammar rejection.", + "nonEnforcing": "Warning / info / advisory / result-shape rules. Their oracle is weaker than a clean rejection (an advisory rule's query SUCCEEDS, so the contract asserts a result shape or mere acceptance), which makes them likelier to move for reasons unrelated to the lint rule \u2014 dedup-consecutive, for instance, depends on the Calcite-to-v2 fallback staying enabled. They ran nightly-only until every contract moved to the PR schedule so the multi-version rollup sees a full trigger census on each PR; they now block like any other contract, and a red result here warrants checking the oracle before editing a rule." } } diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index e1e6a1439d6..688d3439cc7 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -49,6 +49,12 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j | `workflow_dispatch` (`osd_ref`) | OSD-branch evidence | the given commit/branch | No — pre-merge evidence only | | `schedule` (nightly) | full corpus + coverage | `main` | No | +Every contract declares `schedule: "pr"`, so a PR run exercises the **whole corpus** +— 11 rules, 35 queries. A contract that runs also asserts: neither the IT nor the +detector runner consults the manifest's `enforced` list, so any contract on the PR +schedule can fail the required check. Keep that in mind when adding one; a new +contract whose oracle has not settled should say `schedule: "nightly"` until it has. + `workflow_dispatch` inputs: - `osd_repo` — the OSD repository to check out, for validating an unmerged change @@ -184,8 +190,17 @@ rule cannot be validated end to end. single-version `enforced` set because their backend oracle is a semantic `Field [...] not found.` rejection they share with each other rather than a rule-unique grammar rejection. -- `nonEnforcing` — warning/info/advisory/result-shape rules. They run on the - nightly schedule for coverage and never block a PR. +- `nonEnforcing` — warning/info/advisory/result-shape rules. Their oracle is weaker + than a clean rejection: an advisory rule's query *succeeds*, so the contract can + only assert a result shape or plain acceptance, which is likelier to move for + reasons unrelated to the lint rule (`dedup-consecutive` depends on the + Calcite-to-v2 fallback staying on). These ran nightly-only until every contract + moved to the PR schedule, so they now block like any other. A red result here is + worth checking against the oracle before editing a rule. + +The `enforced` / `nonEnforcing` split therefore describes **oracle quality and review +status, not blocking behavior** — it tells a reviewer how much to trust a red result, +not whether one can occur. ## Multi-version validation From 6f1d6dd3a89e7a9261882938b4cdd635758e7278 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 10:48:22 -0700 Subject: [PATCH 26/39] fix(ci): make the connectivity probe actually run The probe reported nothing: its step logged BUILD SUCCESSFUL in 2m45s with no probe output at all. It used JUnit 4's org.junit.Test while this module runs useJUnitPlatform(), so the class was collected as zero tests and the task succeeded vacuously -- the same shape of failure the PPL lint contract itself guards against, in the diagnostic meant to explain it. Switch to org.junit.jupiter.api.Test, matching PplLintRuleValidationIT. Also stop the step's grep from hiding evidence: add --info and match 'tests completed' and 'No tests found' so a zero-test run is visible next time instead of reading as a pass. Signed-off-by: Hanyu Wei --- .github/workflows/ppl-lint-multiversion-validation.yml | 3 ++- .../sql/calcite/remote/RestClientConnectivityProbeIT.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index 19eae573de0..d7ec99ba51d 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -431,7 +431,8 @@ jobs: --tests 'org.opensearch.sql.calcite.remote.RestClientConnectivityProbeIT' \ -Dtests.rest.cluster=localhost:9200 \ -Dtests.cluster=localhost:9200 \ - -Dtests.clustername=docker-cluster 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD' || true + -Dtests.clustername=docker-cluster \ + --info 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD|tests? completed|No tests found' || true - name: Run contract observation against engine ${{ matrix.version }} run: | diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java index 0be3228ff4d..6ca497986ce 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -14,7 +14,7 @@ import java.util.List; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.util.Timeout; -import org.junit.Test; +import org.junit.jupiter.api.Test; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; From f70929e2cecd2a6cc0d4abdd1da5520d433dfafe Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:00:42 -0700 Subject: [PATCH 27/39] fix(ppl-lint): scope the trigger differential to rules the engine actually rejects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving every contract onto the PR schedule turned the required check red with 8 failures, 7 of them this: the trigger cross-check paired the detector against `be.rejected`, which is only meaningful for a `rejection`-kind rule. An advisory rule flags a query the engine runs happily — head-without-sort marks non-determinism, division-by-zero marks a silent null, dedup-consecutive succeeds via the Calcite-to-v2 fallback. For those, "detector flagged, backend accepted" is the rule working as designed, so the check failed every advisory trigger unconditionally, including ones that predate this branch. That, not runtime cost, is the structural reason those contracts could only ever run nightly; I had attributed it to cost and weaker oracles, which was wrong. The check now runs only when the contract declares `backend.kind: "rejection"`. The contracts already carry that distinction, so this reads data that exists rather than adding a flag, and rejection rules are completely unaffected. Advisory triggers keep full coverage from the two other assertions, which is why relaxing the pairing is safe rather than merely convenient: - the backend-kind check still fires if the engine starts REJECTING a query the contract pinned as accepted; - the `detectorCount` assertion still fires if the detector stops flagging it. Every trigger in the corpus pins detectorCount: 1 regardless of kind, so a silent advisory detector is still caught. Verified by replaying the failed CI run's own artifacts (backend-report.json, target.json, ppl-grammar-bundle.json downloaded from run 30289514275): same inputs go from 6 failures to 0 on the compiled surface. Confirmed still selective by tampering the backend report to make `disabled-join-type/right-join-disabled` — a rejection rule — look accepted: that fails with both the backend-kind and the trigger assertion. Signed-off-by: Hanyu Wei --- scripts/ppl-lint/run-frontend-contract.mjs | 28 ++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 6f7ecd6de50..dcdfe5e4a1d 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -685,14 +685,38 @@ function main() { `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` ); } - // Trigger/control cross-check against the detector's own verdict. + // Trigger cross-check: a trigger the detector flags must be one the engine + // ALSO objects to — but only where the contract claims the engine objects + // at all. + // + // For a `rejection` rule the two coincide: detector flags <-> engine + // rejects, and a disagreement means one side drifted. That is the original + // check and it is unchanged. + // + // An ADVISORY rule is different by design. It flags a query the engine + // runs happily: `head-without-sort` marks non-determinism, + // `division-by-zero` marks a silent null, `dedup-consecutive` succeeds via + // the Calcite-to-v2 fallback. "Detector flagged, backend accepted" is that + // rule working, not drift — so pairing the detector against `be.rejected` + // failed every advisory trigger unconditionally. That, not runtime cost, + // is the structural reason those contracts could only run nightly. + // + // The contracts already carry the distinction in `backend.kind`, so this + // reads data that exists rather than adding a flag. Advisory triggers keep + // full coverage from the other two assertions: the backend-kind check above + // fires if the engine starts REJECTING a query pinned as accepted, and the + // `detectorCount` assertion fires if the detector stops flagging it. Only + // the pairing rule is scoped to the rules it makes sense for. const detectorFlagged = actual > 0; - if (role === 'trigger' && detectorFlagged !== !!be.rejected) { + if (role === 'trigger' && expectRejected && detectorFlagged !== !!be.rejected) { failures.push( `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + `but backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` ); } + // A control must pass on both sides regardless of kind: it is a valid + // query the rule has to stay quiet on. Unlike a trigger, that claim does + // not vary with `backend.kind`. if (role === 'control' && (detectorFlagged || be.rejected)) { failures.push( `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + From e5dec9df0e9d3d1e00c7f4dc1fad5e0098b0d439 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:00:59 -0700 Subject: [PATCH 28/39] fix(test): make the connectivity probe discoverable Second vacuous pass from the same probe, different cause. The JUnit 5 switch was wrong: integTestRemote runs the default JUnit 4 runner -- only integJdbcTest calls useJUnitPlatform() -- so the jupiter @Test made the class undiscoverable and Gradle reported 'No tests found for given includes: [**/*IT.class]' while the step still looked fine. But the JUnit 4 annotation alone was not enough either. Gradle only discovers an IT that inherits a runner from a framework base class; a standalone class has none and is collected as zero tests. That is why the FIRST version reported nothing despite compiling, matching the include pattern, and having its .class file in place. Extend OpenSearchTestCase: it supplies the randomized-testing runner but builds no REST client, so the probe is discovered without inheriting the client setup that hangs -- which was the whole reason for not extending the REST base class. Verified locally against a live cluster, all four layers reporting: tcp connect OK in 2ms HttpURLConnection _nodes/plugins OK in 8ms: HTTP 200, 9456 bytes RestClient _nodes/plugins OK in 54ms: HTTP 200, 9456 bytes RestClient _plugins/_ppl/_grammar OK in 20ms: HTTP 200, 248625 bytes Note for anyone running it by hand: the opensearch.rest-test plugin requires tests.rest.cluster, tests.cluster and tests.clustername to be all-null or all-non-null, or the project fails to configure before any test runs. Signed-off-by: Hanyu Wei --- .../remote/RestClientConnectivityProbeIT.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java index 6ca497986ce..0a800fe94a1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -14,11 +14,12 @@ import java.util.List; import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.util.Timeout; -import org.junit.jupiter.api.Test; +import org.junit.Test; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.RestClient; import org.opensearch.client.RestClientBuilder; +import org.opensearch.test.OpenSearchTestCase; /** * Connectivity probe for {@code tests.rest.cluster}, used to diagnose why the REST test client @@ -33,10 +34,14 @@ * negotiation, FIPS, plugin set, port collision, address family) have each been refuted by * observation. * - *

This class deliberately does NOT extend the test framework's base class: that base class is - * what hangs, so inheriting it would reproduce the symptom without isolating the cause. Instead it - * walks up the stack one layer at a time against the same address, so a single run says exactly - * which layer stops working: + *

It extends {@link OpenSearchTestCase}, NOT the REST base class. Two constraints meet here: + * {@code integTestRemote} runs the default JUnit 4 runner (only {@code integJdbcTest} calls {@code + * useJUnitPlatform()}), and Gradle only discovers an IT that inherits a runner from a framework + * base class — a standalone class is silently collected as ZERO tests, which is how the first + * version of this probe "passed" while reporting nothing. {@code OpenSearchTestCase} supplies that + * runner but builds no REST client, so discovery works without inheriting the hang under + * investigation. Instead of the framework's client setup, this walks up the stack one layer at a + * time against the same address, so a single run says exactly which layer stops working: * *

    *
  1. raw TCP connect — is the port reachable from this JVM at all? @@ -52,7 +57,7 @@ *

    Run with: {@code ./gradlew :integ-test:integTestRemote --tests * '*RestClientConnectivityProbeIT' -Dtests.rest.cluster=localhost:9200} */ -public class RestClientConnectivityProbeIT { +public class RestClientConnectivityProbeIT extends OpenSearchTestCase { /** Bound well below the framework's 60s so a hang is visibly a hang, not a wait. */ private static final Timeout PROBE_TIMEOUT = Timeout.ofSeconds(15); From 252876d1c408feac008ba9be7852794d2c6c8a64 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:14:53 -0700 Subject: [PATCH 29/39] test(ci): validate the latest patch of each line, and probe client variants Two fixes. 1. Test the LATEST PATCH, not .0. The compiled matrix pinned 2.19.0 while the published line is at 2.19.6 -- six patches stale, so it validated an engine no user runs and would have attributed any bug fixed in between to the whole 2.19 line. Checked every line on Docker Hub: 2.19.6 is the only stale pin; 3.0.0 / 3.5.0 / 3.6.0 / 3.7.0 already are their lines' latest patch. Comments now say to keep them latest rather than .0. 2. Probe client variants. The previous probe localized the 2.19 timeout precisely: tcp connect OK 4ms HttpURLConnection _nodes/plugins OK 27ms HTTP 200, 15844 bytes RestClient _nodes/plugins FAILED 15396ms SocketTimeoutException RestClient _cluster/health FAILED 15025ms SocketTimeoutException Every endpoint fails, including a 459-byte health response, while the JDK's own HTTP stack succeeds against the same URL. So the fault is in how the async client speaks to this engine -- not the network, engine, response size, or any one endpoint, which is why seven log-derived theories all missed it. Rather than guess again, run candidate configurations side by side against the same endpoint: FORCE_HTTP_1, NEGOTIATE, FORCE_HTTP_2, and a fresh single- connection manager. Whichever succeeds names the fix; if none do, the client cannot be configured around it and the answer is a client/engine version constraint instead. Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 9 ++- .../remote/RestClientConnectivityProbeIT.java | 78 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index d7ec99ba51d..f765c0d132e 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -84,6 +84,9 @@ env: # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must # be >= 3.6.0 (the _grammar endpoint floor) and must have a published # distribution image. + # Latest patch of each line (3.6.0 and 3.7.0 ARE the latest patches today; bump + # them when 3.6.1 / 3.7.1 publish, and never pin `.0` once a newer patch + # exists — that would validate an engine no user runs). ENGINE_VERSIONS: '["3.6.0","3.7.0"]' # Released engine versions to validate on the COMPILED-SIMPLIFIED surface. # @@ -97,7 +100,11 @@ env: # Only contracts declaring `grammarSurface: "both"` are scored here; the rest are # reported not-applicable. Nightly only — see the `compiled_versions` input to # run one ad hoc. - COMPILED_ENGINE_VERSIONS: '["2.19.0","3.0.0","3.5.0"]' + # + # Always the LATEST PATCH of each line, never `.0`. A user on 2.19 is on + # 2.19.6, so validating 2.19.0 tests an engine nobody runs and attributes any + # bug fixed in between to the whole line. + COMPILED_ENGINE_VERSIONS: '["2.19.6","3.0.0","3.5.0"]' jobs: # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java index 0a800fe94a1..517503cc148 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/RestClientConnectivityProbeIT.java @@ -89,6 +89,7 @@ public void probeConnectivity() { boolean tcpOk = probeRawSocket(host, port); probeHttpUrlConnection(host, port); probeRestClient(host, port); + probeClientVariants(host, port); // Only a hard failure here is fatal: without a socket the rest is noise. if (!tcpOk) { @@ -201,6 +202,83 @@ private static String describe(Throwable error) { return String.join(" <- ", chain); } + /** + * Layer 4: candidate fixes, each a one-line change from the default client, all against the same + * endpoint on the same engine. + * + *

    The default async client times out on EVERY endpoint here — including a 459-byte {@code + * _cluster/health} — while {@code HttpURLConnection} against the same URL returns 200 in 27ms. So + * the fault is in how the async client speaks to this engine, not in the network, the engine, the + * response size, or any one endpoint. Each variant isolates one suspect; whichever succeeds names + * the fix, and if none do, the client cannot be configured around it. + */ + private void probeClientVariants(String host, int port) { + // Forcing HTTP/1.1 up front, rather than letting HttpClient 5.x negotiate h2. + variant( + host, + port, + "FORCE_HTTP_1", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.FORCE_HTTP_1))); + // Same, but negotiating h2 explicitly, to tell "policy matters" from "1.1 specifically works". + variant( + host, + port, + "NEGOTIATE", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.NEGOTIATE))); + // FORCE_HTTP_2, to confirm the direction of any protocol effect rather than assume it. + variant( + host, + port, + "FORCE_HTTP_2", + b -> + b.setHttpClientConfigCallback( + c -> c.setVersionPolicy(org.apache.hc.core5.http2.HttpVersionPolicy.FORCE_HTTP_2))); + // A fresh connection manager: rules out connection reuse/pooling against this engine. + variant( + host, + port, + "fresh-conn-manager", + b -> + b.setHttpClientConfigCallback( + c -> + c.setConnectionManager( + org.apache.hc.client5.http.impl.nio + .PoolingAsyncClientConnectionManagerBuilder.create() + .setMaxConnPerRoute(1) + .setMaxConnTotal(1) + .build()))); + } + + /** Run one client variant against {@code _cluster/health} — the smallest response available. */ + private void variant( + String host, + int port, + String name, + java.util.function.UnaryOperator tune) { + RestClientBuilder builder = + RestClient.builder(new HttpHost("http", host, port)) + .setRequestConfigCallback( + config -> config.setConnectTimeout(PROBE_TIMEOUT).setResponseTimeout(PROBE_TIMEOUT)) + .setStrictDeprecationMode(false); + long start = System.nanoTime(); + try (RestClient client = tune.apply(builder).build()) { + Response response = client.performRequest(new Request("GET", "/_cluster/health")); + log( + "variant " + + name + + " OK in " + + millis(start) + + "ms: HTTP " + + response.getStatusLine().getStatusCode()); + } catch (Exception e) { + log("variant " + name + " FAILED after " + millis(start) + "ms: " + describe(e)); + } + } + private static long millis(long startNanos) { return (System.nanoTime() - startNanos) / 1_000_000; } From 19696487865e64e8e079203d0158d34dfeb51d25 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:36:07 -0700 Subject: [PATCH 30/39] fix(test): tolerate a pre-Calcite engine in observe-only mode With the version pin corrected to 2.19.6, the leg gets past the client entirely and fails on the real incompatibility instead: PUT /_cluster/settings -> 400 persistent setting [plugins.calcite.enabled], not recognized Calcite is a 3.x feature and SQLIntegTestCase.init() sets that setting unconditionally, so every 2.x leg aborts before seeding a fixture or running a query. In observe-only mode, catch exactly that failure and continue without the setting: a pre-Calcite engine is a legitimate thing to observe, and each contract's own frontendContext.isCalcite already states what the linter should assume there. super.init() aborts partway when it throws, so redo the version-independent half (increaseMaxCompilationsRate). Matched narrowly -- on the setting name plus "not recognized", not on any 400 -- so a genuinely broken settings call on a Calcite-capable engine still fails rather than being waved through as "old engine". Asserting mode is unchanged: the required check runs against the PR's own build, where a missing Calcite setting is a real problem. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 52 +++++++++++++++++-- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 6a2de542ab6..336bff4bf68 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -64,8 +64,8 @@ *

    The suite honors {@code -Dppl.lint.schedule=pr|nightly} (default {@code pr}): a PR run skips * contracts declaring {@code schedule: "nightly"}, while nightly runs the full corpus. Every * contract in the corpus currently declares {@code schedule: "pr"}, so the two are equivalent - * today; the filter stays because it is the only mechanism for holding a new contract back from - * PR runs while its oracle is still settling. + * today; the filter stays because it is the only mechanism for holding a new contract back from PR + * runs while its oracle is still settling. * *

    Note that a contract which RUNS also ASSERTS. This class does not consult the manifest's * {@code enforced} list — that list records oracle quality and review status, not blocking @@ -111,8 +111,29 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { @Override public void init() throws Exception { - super.init(); - enableCalcite(); + // Calcite is a 3.x engine feature: on 2.x the cluster rejects + // `plugins.calcite.enabled` outright with "not recognized", and the base class's + // init() sets it unconditionally. In observe-only mode (the multi-version + // matrix) that must not abort the leg — a pre-Calcite engine is a legitimate + // thing to observe, and the contracts' own `frontendContext.isCalcite` already + // describes what the linter should assume there. + // + // Asserting mode keeps the strict behavior: the required check runs against the + // PR's own build, where a missing Calcite setting is a real problem. + try { + super.init(); + enableCalcite(); + } catch (Exception e) { + if (!observeOnly || !isUnrecognizedCalciteSetting(e)) { + throw e; + } + System.err.println( + "[ppl-lint] engine does not support the Calcite setting; observing without it: " + + e.getMessage()); + // super.init() aborted partway, so redo the part that is version-independent. + increaseMaxCompilationsRate(); + } + // Fall through to fixture seeding either way. // Seed the union of every index every scheduled contract needs, once. for (String indexEnum : requiredIndexEnums()) { try { @@ -639,6 +660,29 @@ private void exportGrammarArtifacts(List failures) { } /** Target manifest for a leg with no grammar bundle (compiled surface / local run). */ + /** + * True when a failure is the cluster rejecting {@code plugins.calcite.enabled} because it does + * not know that setting — i.e. a pre-Calcite (2.x) engine. + * + *

    Deliberately narrow: matched on the setting name plus "not recognized" rather than on any + * 400, so a genuinely broken settings call on a Calcite-capable engine still fails the run + * instead of being waved through as "old engine". + */ + private static boolean isUnrecognizedCalciteSetting(Throwable error) { + for (Throwable current = error; current != null; current = current.getCause()) { + String message = current.getMessage(); + if (message != null + && message.contains(Settings.Key.CALCITE_ENGINE_ENABLED.getKeyValue()) + && message.contains("not recognized")) { + return true; + } + if (current.getCause() == current) { + break; + } + } + return false; + } + private void writeTargetManifest(String grammarHash, List failures) { writeTargetManifest(grammarHash, "", failures); } From e46d1ec62610c9df57b8f421935b6d86cc02b294 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 11:49:03 -0700 Subject: [PATCH 31/39] fix(test): skip the whole Calcite settings family on a pre-Calcite engine The init() tolerance worked -- the log shows 'observing without it' -- but the leg still failed, because applyClusterSettings re-applies the same setting per contract from backendFixture.clusterSettings, undoing it once per contract. Every setting in that block is Calcite-family (calcite, calciteFallback, allJoinTypesAllowed), and a pre-Calcite engine rejects all of them identically. So record support once in init() and skip the block, rather than wrapping each call in the same catch. Verified in the same run that the earlier fix landed: 2.19.6's REST client is healthy (_nodes/plugins 312ms, _cluster/health 6ms, FORCE_HTTP_1 / NEGOTIATE / fresh-conn-manager all OK; only FORCE_HTTP_2 fails, correctly, since 2.19 has no h2). There was never a client bug -- 2.19.0 was six patches stale. Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 336bff4bf68..7ed5e6ad989 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -102,6 +102,14 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private int[] clusterVersion; private String engineVersionRaw; + /** + * Whether this cluster recognizes the Calcite settings at all. False on a pre-Calcite (2.x) + * engine, where every {@code plugins.calcite.*} write is rejected as "not recognized". + * Established once in {@code init()} and honored by {@code applyClusterSettings}, so the whole + * family is skipped rather than retried and failed once per contract. + */ + private boolean calciteSettingsSupported = true; + /** * Index fixtures that could not be created on this engine (observe-only mode only). Contracts * that need one are reported as {@code outcome: "error"} instead of as engine behavior, because @@ -127,8 +135,9 @@ public void init() throws Exception { if (!observeOnly || !isUnrecognizedCalciteSetting(e)) { throw e; } + calciteSettingsSupported = false; System.err.println( - "[ppl-lint] engine does not support the Calcite setting; observing without it: " + "[ppl-lint] engine does not support the Calcite settings; observing without them: " + e.getMessage()); // super.init() aborted partway, so redo the part that is version-independent. increaseMaxCompilationsRate(); @@ -738,6 +747,13 @@ private List applyClusterSettings(JSONObject fixture) throws IOException if (settings == null) { return applied; } + // Every setting below is Calcite-family, and a pre-Calcite engine rejects all of + // them the same way. `init()` already established whether this cluster knows + // them, so skip the whole block rather than fail per contract — otherwise the + // tolerance added there is undone here, once per contract. + if (!calciteSettingsSupported) { + return applied; + } if (settings.has("calcite")) { if (settings.getBoolean("calcite")) { enableCalcite(); From 5a3cd066f15a150285c7d7ddb1569875702c15ac Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 13:07:09 -0700 Subject: [PATCH 32/39] revert(test): un-pin the REST client from HTTP/1.1 Reverts 4c80f7b91. That commit claimed HTTP/2 negotiation as the cause of the 2.19 observation-leg timeout. It was not, and the pin fixed nothing. The real cause was the version pin: the leg tested opensearchproject/opensearch: 2.19.0, six patches behind the 2.19.6 the line actually ships. On 2.19.6 the unmodified client is healthy -- _nodes/plugins in 312ms, _cluster/health in 6ms -- and the variant probe confirms it is not protocol-related at all: FORCE_HTTP_1 OK 18ms NEGOTIATE OK 21ms fresh-conn-manager OK 12ms FORCE_HTTP_2 FAILED (correctly: 2.19 has no h2) Since NEGOTIATE -- the default -- works, forcing 1.1 was a no-op dressed as a fix, and leaving it in would have suggested a protocol constraint that does not exist. Signed-off-by: Hanyu Wei --- .../sql/legacy/OpenSearchSQLRestTestCase.java | 44 +++---------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java index 0abe1f4ab7c..267adea43c3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/OpenSearchSQLRestTestCase.java @@ -18,7 +18,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; -import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.nio.PoolingAsyncClientConnectionManagerBuilder; import org.apache.hc.client5.http.ssl.ClientTlsStrategyBuilder; @@ -29,7 +28,6 @@ import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.message.BasicHeader; import org.apache.hc.core5.http.nio.ssl.TlsStrategy; -import org.apache.hc.core5.http2.HttpVersionPolicy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.util.Timeout; import org.apache.logging.log4j.LogManager; @@ -257,39 +255,12 @@ protected static void configureClient(RestClientBuilder builder, Settings settin credentialsProvider.setCredentials( new AuthScope(null, -1), new UsernamePasswordCredentials(userName, password.toCharArray())); - return forceHttp11( - httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); }); - } else { - builder.setHttpClientConfigCallback(OpenSearchSQLRestTestCase::forceHttp11); } OpenSearchRestTestCase.configureClient(builder, settings); } - /** - * Pin a client to HTTP/1.1. - * - *

    {@code RestClientBuilder} builds its async client with no version policy, which in - * HttpClient 5.x means "negotiate h2". Against a server that supports h2 that is fine; against - * one that does not, the async I/O reactor stalls instead of falling back, so every request fails - * with {@code SocketTimeoutException} after the full response timeout — thrown from {@code - * AbstractSingleCoreIOReactor.execute} before a single test runs. - * - *

    Live-verified on the PPL lint multi-version matrix: an {@code --http2} probe against engine - * 3.5.0 negotiated HTTP/2 and that leg PASSED, while the same probe against 2.19.0 reported - * HTTP/1.1 and the leg timed out at exactly 60s. curl falls back cleanly; this client does not. - * These tests never need h2, so asking for 1.1 up front removes the negotiation and works across - * every supported engine line. - * - *

    Applied INSIDE each config callback rather than as its own {@code - * setHttpClientConfigCallback} call, because that setter replaces rather than accumulates: a - * separate call would silently drop the credentials or TLS configuration set here, and only on - * the paths where it matters. - */ - private static HttpAsyncClientBuilder forceHttp11(HttpAsyncClientBuilder httpClientBuilder) { - return httpClientBuilder.setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_1); - } - protected static void configureHttpsClient( RestClientBuilder builder, Settings settings, HttpHost httpHost) throws IOException { Map headers = ThreadContext.buildDefaultHeaders(settings); @@ -321,13 +292,12 @@ protected static void configureHttpsClient( .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) .build(); - return forceHttp11( - httpClientBuilder - .setDefaultCredentialsProvider(credentialsProvider) - .setConnectionManager( - PoolingAsyncClientConnectionManagerBuilder.create() - .setTlsStrategy(tlsStrategy) - .build())); + return httpClientBuilder + .setDefaultCredentialsProvider(credentialsProvider) + .setConnectionManager( + PoolingAsyncClientConnectionManagerBuilder.create() + .setTlsStrategy(tlsStrategy) + .build()); } catch (Exception e) { throw new RuntimeException(e); } From 042a123714e05df2263532865a62c94192ccbbf7 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Mon, 27 Jul 2026 13:23:49 -0700 Subject: [PATCH 33/39] test(ppl-lint): pin dense_rank's 3.7 rejection wording The multi-version check flagged engine-message-changed on 3.7.0: error.reason "There was internal problem at backend" -> "Unexpected window function: dense_rank" The >=3.7.0 <3.8.0 expectation for eventstats-dense-rank was copied from the pre-3.7 epoch and never updated, while its sibling eventstats-rank in the SAME epoch already pins the function-naming wording. 3.7 names the offending function; only the pre-3.7 engines emit the generic message. Detector-side verdict is unaffected, so this is the update-contract remediation the classifier recommended -- not a rule change. Signed-off-by: Hanyu Wei --- .../unsupported-window-function-in-eventstats.spec.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json index 6479fd2b1b1..75e45299c0f 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/unsupported-window-function-in-eventstats.spec.json @@ -117,7 +117,7 @@ "status": 500, "error": { "type": "UnsupportedOperationException", - "reason": "There was internal problem at backend" + "reason": "Unexpected window function: dense_rank" } } } From 60f86729d121ca5e8bbc79b4a797ef4fb1932e5b Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 12:35:48 -0700 Subject: [PATCH 34/39] feat(ci): observe PPL lint contracts on analytics engine Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 105 +- .../workflows/ppl-lint-rule-validation.yml | 9 + ...ppl-lint-analytics-engine-ci-validation.md | 811 +++++++++++++ integ-test/build.gradle | 54 +- .../remote/PplLintRuleValidationIT.java | 1048 +++++++++++++++-- .../contracts/division-by-zero.spec.json | 11 +- scripts/ppl-lint-rule-validation.sh | 40 +- scripts/ppl-lint/README.md | 65 +- .../__tests__/aggregate-versions.test.mjs | 752 +++++++++++- scripts/ppl-lint/__tests__/annotate.test.mjs | 23 + .../__tests__/assemble-run-manifest.test.mjs | 174 +++ .../__tests__/contract-schema.test.mjs | 469 ++++++++ scripts/ppl-lint/__tests__/drift.test.mjs | 74 ++ scripts/ppl-lint/aggregate-versions.mjs | 982 ++++++++++++--- scripts/ppl-lint/annotate.mjs | 33 +- scripts/ppl-lint/assemble-run-manifest.mjs | 158 ++- scripts/ppl-lint/contract-schema.mjs | 397 +++++++ scripts/ppl-lint/drift.mjs | 278 ++++- scripts/ppl-lint/probe-discovery-backend.mjs | 7 +- scripts/ppl-lint/run-frontend-contract.mjs | 348 ++++-- 20 files changed, 5459 insertions(+), 379 deletions(-) create mode 100644 docs/dev/ppl-lint-analytics-engine-ci-validation.md create mode 100644 scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs create mode 100644 scripts/ppl-lint/__tests__/contract-schema.test.mjs create mode 100644 scripts/ppl-lint/contract-schema.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index f765c0d132e..b02a2a5ecf7 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -55,7 +55,10 @@ on: # point is to see the multi-version effect of the change. pull_request: paths: + - 'integ-test/build.gradle' + - 'integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java' - 'integ-test/src/test/resources/ppl-lint/**' + - 'scripts/ppl-lint-rule-validation.sh' - 'scripts/ppl-lint/**' - '.github/workflows/ppl-lint-multiversion-validation.yml' workflow_dispatch: @@ -80,6 +83,10 @@ on: permissions: contents: read +concurrency: + group: ppl-lint-multiversion-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + env: # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must # be >= 3.6.0 (the _grammar endpoint floor) and must have a published @@ -273,6 +280,8 @@ jobs: -Dtests.clustername=docker-cluster \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha="${GITHUB_SHA}" \ -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ -Dppl.lint.grammar.bundle="$(pwd)/leg/ppl-grammar-bundle.json" \ -Dppl.lint.target="$(pwd)/leg/target.json" @@ -454,6 +463,8 @@ jobs: -Dtests.clustername=docker-cluster \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha="${GITHUB_SHA}" \ -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ -Dppl.lint.target="$(pwd)/leg/target.json" # Mark the leg so the detect job knows to lint it on the compiled surface. @@ -513,6 +524,8 @@ jobs: --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ -Dppl.lint.report=$(pwd)/leg/backend-report.json \ -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ -Dppl.lint.target=$(pwd)/leg/target.json" @@ -533,6 +546,63 @@ jobs: name: ppl-lint-leg-pr-build-logs path: | integ-test/build/reports/** + integ-test/build/test-results/** + integ-test/build/testclusters/*/logs/* + + # The PR build through the full composite/Parquet + DataFusion stack. This is + # an observation leg: route/identity/infrastructure failures are fatal, while + # backend oracles are promoted only after their captured behavior is reviewed. + observe-pr-build-analytics: + name: Observe engine pr-build (analytics) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 25 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 25 + + - name: Run analytics contract observation against the PR build + run: | + set -euo pipefail + mkdir -p leg + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ + -Dppl.lint.report=$(pwd)/leg/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/leg/target.json" + + - name: Upload analytics leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-pr-build-analytics + path: leg + if-no-files-found: error + + - name: Upload analytics failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-pr-build-analytics-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** integ-test/build/testclusters/*/logs/* # Lint each engine's exported grammar with the OSD detectors. Separate from the @@ -549,6 +619,7 @@ jobs: - observe-released - observe-compiled - observe-pr-build + - observe-pr-build-analytics if: ${{ always() && needs.plan.result == 'success' }} runs-on: ubuntu-latest timeout-minutes: 40 @@ -639,7 +710,11 @@ jobs: echo "skipping $leg (no grammar bundle and no compiled-surface marker)" continue fi - env "${surface_env[@]}" \ + observe_env=(PPL_LINT_OBSERVE_ONLY=1) + if [ "$(jq -r '.executionBackend // empty' "$leg/target.json")" = 'analytics' ]; then + observe_env+=(PPL_LINT_OBSERVE_ANALYTICS=1) + fi + env "${surface_env[@]}" "${observe_env[@]}" \ PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ PPL_LINT_SCHEDULE=nightly \ PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ @@ -693,7 +768,7 @@ jobs: import json,sys print(' '.join(f'{v}-compiled' for v in json.load(sys.stdin))) ") - for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build; do + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build pr-build-analytics; do found=no for have in "${present[@]}"; do [ "$have" = "$want" ] && found=yes && break @@ -708,6 +783,7 @@ jobs: --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ --out "$GITHUB_WORKSPACE/drift-report.json" \ --summary "$GITHUB_STEP_SUMMARY" \ + --observe-analytics \ "${args[@]}" - name: Upload drift report @@ -876,8 +952,13 @@ jobs: ") python3 -c " import json - json.dump({'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', - 'grammarHash': '$hash'}, + json.dump({'schemaVersion': 2, + 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '$hash', + 'grammarBundle': 'discovery-bundle.json', + 'executionBackend': 'standard', + 'storage': 'lucene', + 'shardCount': 1}, open('$GITHUB_WORKSPACE/discovery-target.json','w')) " echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" @@ -888,6 +969,17 @@ jobs: # The surface is recorded in the report, so a reader can see which ran. echo "::warning::_grammar export failed; falling back to the compiled surface (runtimeOnly rules will not be observed)." echo "surface=compiled-simplified" >> "$GITHUB_OUTPUT" + python3 -c " + import json + json.dump({'schemaVersion': 2, + 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '', + 'grammarBundle': '', + 'executionBackend': 'standard', + 'storage': 'lucene', + 'shardCount': 1}, + open('$GITHUB_WORKSPACE/discovery-target.json','w')) + " fi - name: Run the detectors over the discovery corpus @@ -900,10 +992,11 @@ jobs: # expanding an empty array as "${a[@]}" is an unbound-variable error in bash # before 4.4, which would crash the compiled-surface fallback — the very # path that only runs when something else already went wrong. - extra=(PPL_LINT_DISCOVERY=1) + extra=(PPL_LINT_DISCOVERY=1 + PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") if [ "$SURFACE" = 'runtime-bundle' ]; then extra+=(PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" - PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") + ) fi # A non-zero exit is EXPECTED and ignored: the generated specs carry # placeholder expectations, so the runner reports a "failure" for every diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 543139e3ebb..078fd285494 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -1,5 +1,12 @@ name: PPL lint rule validation +permissions: + contents: read + +concurrency: + group: ppl-lint-rule-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + # Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint detectors and # the SQL backend must agree on the SAME candidate runtime grammar. A shared, # reviewed corpus of contract files pins each rule's OSD detector diagnostic @@ -134,6 +141,8 @@ jobs: su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ -Dppl.lint.report=$(pwd)/backend-report.json \ -Dppl.lint.grammar.bundle=$(pwd)/ppl-grammar-bundle.json \ -Dppl.lint.target=$(pwd)/target.json" diff --git a/docs/dev/ppl-lint-analytics-engine-ci-validation.md b/docs/dev/ppl-lint-analytics-engine-ci-validation.md new file mode 100644 index 00000000000..3777f561dc7 --- /dev/null +++ b/docs/dev/ppl-lint-analytics-engine-ci-validation.md @@ -0,0 +1,811 @@ +# Analytics Engine Coverage for PPL Lint CI Validation + +- **Status:** Validated for phased implementation +- **Last updated:** 2026-07-28 +- **Scope:** PPL lint contract validation in + `.github/workflows/ppl-lint-rule-validation.yml` and + `.github/workflows/ppl-lint-multiversion-validation.yml` + +## 1. Summary + +The PPL lint CI contract currently compares OpenSearch Dashboards (OSD) +detectors with the standard SQL execution route only. It does not prove that +the same lint diagnostics are correct when a query is routed through the +analytics engine and executed by DataFusion over composite/Parquet storage. + +This design adds analytics-engine coverage by: + +1. Running the existing `PplLintRuleValidationIT` corpus against a dedicated, + full-stack analytics-engine test cluster. +2. Making execution backend an explicit contract and artifact dimension, + separate from OpenSearch version, Calcite applicability, and grammar + surface. +3. Failing if the analytics lane silently falls back to the standard route. +4. Running the OSD detector comparison against both standard and analytics + backend reports while bootstrapping OSD only once. +5. Shipping the lane as non-enforcing observation first, then adding it to the + stable required result after its artifacts, expectations, and reliability + meet the promotion criteria in this document. + +The initial implementation covers the SQL pull request build on one shard. It +does not add a Cartesian product of analytics backends, released OpenSearch +versions, grammar surfaces, and shard counts. + +## 2. Current State + +### 2.1 Required PPL lint validation + +`.github/workflows/ppl-lint-rule-validation.yml` is a three-job pipeline: + +```text +backend-validation + -> detector-validation + -> validation-result +``` + +- `backend-validation` runs `PplLintRuleValidationIT` against the ordinary + Gradle `integTest` cluster. That cluster installs SQL, Job Scheduler, and + Geospatial, but not the analytics-engine stack. +- The integration test executes every scheduled trigger and control query + against `POST /_plugins/_ppl`, then exports: + - `ppl-grammar-bundle.json` + - `target.json` + - `backend-report.json` +- `detector-validation` bootstraps OSD, runs its production headless PPL lint + API against the exported grammar, and compares detector output with the + backend report. +- `validation-result` uses `if: always()` and fails unless both producer jobs + succeeded. This is the stable branch-protection check. + +The multi-version companion workflow repeats the same contract against +released standard engines and the pull request build. Its current dimensions +are OpenSearch version and grammar surface. + +### 2.2 Existing analytics-engine support + +The repository already contains most of the required test infrastructure: + +- `integ-test/build.gradle` can download the analytics engine, Arrow, + composite engine, Parquet data format, and Lucene/DataFusion backend plugin + ZIPs. +- The full analytics stack is already configured for + `analyticsEngineProfileIT` and `analyticsEngineSecurityIT`. +- `-Dtests.analytics.parquet_indices=true` makes helper-created fixtures use + composite/Parquet storage. +- `SQLIntegTestCase` applies the corresponding cluster defaults before fixture + creation. +- `PPLIntegTestCase.isAnalyticsParquetIndicesEnabled()` exposes the active + route to tests. +- `integTestRemote` already forwards the analytics fixture properties. +- `CalciteAnalyticsDatetimeWireFormatIT` demonstrates route attestation using + explain output: analytics plans contain + `LogicalTableScan(table=[[opensearch,` and not + `CalciteLogicalIndexScan`. + +### 2.3 Gap in the existing analytics workflow + +`.github/workflows/analytics-engine-compat.yml` runs only +`AnalyticsEngineCompatIT`. Its purpose is plugin coexistence. Its PPL assertion +uses the `rest` row source, which is explicitly excluded from analytics +routing. The workflow can therefore pass without executing a PPL query through +DataFusion. + +The `analyticsEngineCompat` cluster is also intentionally smaller than the +stack required for real analytics execution. It does not install the composite +engine, Parquet data format, or both analytics backends. + +### 2.4 Terminology + +The following dimensions must remain independent: + +| Dimension | Examples | Meaning | +| --- | --- | --- | +| Engine version | `3.7.0`, `3.8.0-SNAPSHOT` | OpenSearch/SQL product version | +| Grammar surface | `runtime-bundle`, `compiled-simplified` | Grammar used by OSD lint | +| Lint/planner applicability | `engine: "calcite"` | Existing OSD rule applicability | +| Execution backend | `standard`, `analytics` | SQL execution route selected at runtime | +| Storage | `lucene`, `composite-parquet` | Fixture storage that drives routing | + +Analytics uses Calcite planning, so treating `analytics` as another value of +the existing `engine` field would be incorrect. Treating it as another engine +version would also cause the drift analyzer to recommend version scoping for a +backend-specific difference. + +## 3. Problem Statement + +A lint rule is presented to users before query execution. OSD currently has no +analytics-route signal in the lint context, so the same detector result applies +whether the selected index later uses the standard or analytics route. + +The current CI can miss these failures: + +1. A detector reports an error for a query that the analytics backend accepts. + This is a false positive for analytics users. +2. A detector is silent for a query rejected only by the analytics route. This + is a false negative for analytics users. +3. A control query passes on the standard route but fails on analytics. +4. An analytics test is configured incorrectly and silently executes on the + standard route, producing a vacuous green result. +5. Standard and analytics observations are stored under the same product + version, causing aggregation to overwrite or misclassify one of them. +6. A required job consumes mutable `feature-datafusion/latest` artifacts, so a + rerun can test a different stack without recording that change. + +## 4. Goals and Non-Goals + +### 4.1 Goals + +- Run every scheduled PPL lint trigger and control against the pull request's + analytics route. +- Reuse the existing contract corpus and Java integration-test oracle. +- Use byte-identical query text, the same SQL commit, the same runtime grammar, + the same OSD commit, and the same frontend lint context for both backends. +- Represent execution backend in contracts, reports, manifests, summaries, and + aggregation keys. +- Prove that the analytics plugin stack is installed, fixtures are + composite/Parquet, routing selected analytics, and DataFusion executed a + canary query. +- Distinguish backend-route divergence from version drift. +- Fail closed on missing reports, missing expectations, route fallback, + incomplete matrices, or inconsistent grammar identity. +- Produce enough artifacts to reproduce infrastructure and semantic failures. +- Keep pull request wall-clock growth bounded by running backend jobs in + parallel and bootstrapping OSD once. + +### 4.2 Non-goals + +- Replacing the existing broad analytics compatibility, security, or profile + suites. +- Running the entire PPL integration-test suite in the lint validation job. +- Adding browser, Monaco, or a running OSD server. +- Performance or benchmark validation. +- Testing every released OpenSearch version with every analytics stack in the + first release. +- Adding multi-shard analytics coverage to the required lint check. +- Automatically accepting known analytics limitations through broad Gradle + exclusions or JUnit assumptions. +- Changing production routing solely to make the test easier. + +## 5. Design Invariants + +The implementation must preserve these invariants: + +1. **Same SQL candidate:** both backend lanes build the same checked-out SQL + commit. +2. **Same grammar:** both lanes export a runtime bundle. Their engine version + and grammar hash must match before detector validation starts. +3. **Same OSD candidate:** both detector comparisons use one resolved OSD SHA + and one OSD bootstrap. +4. **Same queries:** standard, analytics, and detector passes read the same + contract files and substitute the same index names. +5. **Explicit identity:** every target and report names its execution backend. + Missing or conflicting identity is an infrastructure failure. +6. **Proven route:** setting `tests.analytics.parquet_indices=true` is not + sufficient evidence. The analytics lane must attest the installed plugins, + index settings, explain plan, and a profiled execution. +7. **No semantic retry:** downloads and cluster startup may be retried within + bounded limits. Contract queries and assertions are executed once. +8. **No vacuous pass:** missing queries, reports, detector rows, route evidence, + or planned matrix legs fail or become an explicit non-applicable result. +9. **No implicit fallback:** the analytics lane must never count a standard + route result as analytics coverage. +10. **One detector oracle:** detector count and severity remain route + independent until OSD exposes an execution-backend lint context. +11. **Complete contracts:** every selected expectation names exactly the same + query keys as the contract's top-level `queries` map. Duplicate or missing + report rows are infrastructure failures. +12. **Strict artifacts:** requested targets and reports must exist, parse, and + agree on execution identity. Writers and consumers fail rather than degrade + to an identity-free or differential-free run. + +## 6. Target Identity + +`target.json` currently records only engine version, grammar hash, and bundle +name. It will move to schema version 2 and include execution identity: + +```json +{ + "schemaVersion": 2, + "sqlSha": "...", + "engineVersion": "3.8.0-SNAPSHOT", + "grammarHash": "sha256:...", + "grammarBundle": "ppl-grammar-bundle.json", + "executionBackend": "analytics", + "storage": "composite-parquet", + "shardCount": 1, + "analyticsStack": { + "source": "immutable feature-build URL", + "buildId": "...", + "components": [ + { + "name": "analytics-engine", + "version": "3.8.0-SNAPSHOT", + "sha256": "..." + } + ] + }, + "routeAttestation": { + "pluginsVerified": true, + "clusterSettingsVerified": true, + "fixtureIndicesVerified": true, + "explainVerified": true, + "profiledExecutionVerified": true + } +} +``` + +For the standard route: + +```json +{ + "executionBackend": "standard", + "storage": "lucene", + "shardCount": 1 +} +``` + +The backend report, detector report, drift report, and run manifest will also +carry `executionBackend`. Aggregation keys become: + +```text +(leg label, engine version, grammar surface, execution backend) +``` + +The leg label remains the presentation key because multiple legs can share the +same engine version. + +## 7. Contract Schema + +### 7.1 Schema version 4 + +Detector expectations are shared, while backend oracles are keyed by execution +backend: + +```json +{ + "schemaVersion": 4, + "ruleId": "union-min-datasets", + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { + "role": "trigger", + "query": "union [ source={{index}} ]" + }, + "union-two-datasets-control": { + "role": "control", + "query": "union [ source={{index}} ] [ source={{index}} ]" + } + }, + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException" + } + } + } + } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} +``` + +The existing `engine` field keeps its current meaning. It is not renamed to +avoid mixing this work with an unrelated contract migration. + +### 7.2 Compatibility and migration + +- A schema version 3 `backend` object is read as `backends.standard`. It is not + used as an implicit analytics oracle. +- Observation can begin before every analytics oracle is reviewed. In + observation mode, a missing analytics oracle executes the query once and + records `coverage-missing` plus the raw backend result; it does not score that + result against the standard oracle. Infrastructure and route-attestation + failures still fail the lane. +- Enforcement requires `backends.analytics` for every selected query. +- An unknown execution backend is a contract error. +- An unknown schema version is a contract error. +- More than one version/planner expectation match remains an error. +- A selected expectation must name exactly the top-level contract query set. +- The Java test and Node runner must implement identical selection behavior. + +An explicit non-applicable form is permitted only when the fixture cannot +meaningfully exercise analytics: + +```json +{ + "kind": "not-applicable", + "reason": "Fixture field type cannot be represented by composite/Parquet storage", + "owner": "@analytics-team", + "issue": "https://github.com/opensearch-project/sql/issues/..." +} +``` + +Rules in the required `defaultError` set cannot be promoted while their +analytics oracle is non-applicable. For other rules, non-applicable entries +remain visible in the report and require an owner and issue. + +### 7.3 Differential policy + +| Case | Detector requirement | Backend requirement | +| --- | --- | --- | +| Control | Zero diagnostics | Every applicable backend accepts | +| Rejection trigger | Expected diagnostic count and severity | Every applicable backend rejects with its reviewed error shape | +| Advisory trigger | Expected diagnostic count and severity | Backend matches its reviewed acceptance/result-shape oracle | +| Missing backend oracle | Not scored | Coverage failure | +| Backend transport error | Not scored | Infrastructure/inconclusive failure, never acceptance | + +If an error rule fires while analytics accepts the trigger, the result is +`execution-backend-divergence`. The remediation must not recommend changing an +OpenSearch version range. Because OSD currently lacks backend context, the +choices are to make the rule valid for both routes, narrow the detector to +behavior common to both, disable it, or first add a reliable backend signal to +the OSD lint context. + +## 8. Analytics Test Cluster and Gradle Task + +### 8.1 Chosen approach + +Add a dedicated Gradle-managed cluster and task: + +```text +testClusters.analyticsEnginePplLint +:integ-test:analyticsEnginePplLintIT +``` + +The cluster will install: + +- Job Scheduler +- Arrow Base +- Arrow Flight RPC +- Analytics Engine +- Composite Engine +- Parquet Data Format +- Analytics Backend Lucene +- Analytics Backend DataFusion +- The SQL plugin built from the current checkout + +It will reuse the native-access, Netty, and experimental feature settings used +by the existing full-stack profile/security clusters. Shared cluster +configuration should be extracted into a small Gradle helper if that can be +done without changing those tasks' behavior. + +The task will: + +- Depend on all analytics plugin downloads and SQL `bundlePlugin`. +- Filter to `PplLintRuleValidationIT`. +- Set `tests.analytics.parquet_indices=true`. +- Set `tests.analytics.num_shards=1`. +- Set `ppl.lint.execution_backend=analytics`. +- Forward the existing `ppl.lint.*` paths and schedule. +- Run as a non-root user in CI. + +Example invocation: + +```bash +./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.execution_backend=analytics \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$PWD/leg/backend-report.json" \ + -Dppl.lint.grammar.bundle="$PWD/leg/ppl-grammar-bundle.json" \ + -Dppl.lint.target="$PWD/leg/target.json" +``` + +The task sets the analytics fixture properties itself so a caller cannot +accidentally request an analytics report while creating Lucene fixtures. + +### 8.2 Why not the alternatives + +**Reuse `analyticsEngineCompatIT`:** rejected because its cluster lacks the full +execution stack and its test intentionally avoids analytics routing. + +**Provision an external cluster and use `integTestRemote`:** the remote task is +a valid future path for released analytics stacks, but it requires separate +cluster lifecycle, plugin installation, and SQL-plugin provenance checks. A +managed cluster is simpler and guarantees that the SQL plugin comes from the +current checkout. + +**Run the full PPL integration suite:** rejected for the required lint check. +It adds unrelated capability exclusions, runtime, and flakiness without +improving the detector contract. + +## 9. Route Attestation + +The analytics integration test will perform attestation before scoring any +contract: + +1. Query `/_cat/plugins?format=json` and require every plugin in the full stack. +2. Verify plugin versions are compatible with the OpenSearch/SQL version. +3. Read cluster settings and require composite data format defaults. +4. Read settings for every fixture index and require: + - `index.pluggable.dataformat.enabled=true` + - `index.pluggable.dataformat=composite` + - `index.composite.primary_data_format=parquet` +5. Run one valid explain canary per fixture index: + + ```text + source= | head 1 + ``` + + Require `LogicalTableScan(table=[[opensearch,` and reject + `CalciteLogicalIndexScan`. +6. Run the same canary with `profile=true` and require at least one successful + execution stage. Record every `execution_type`. Before required promotion, + pin and require the exact DataFusion-specific marker exposed by the locked + analytics stack. A generic non-empty profile is sufficient only for the + observation lane. +7. Write the attestation outcome into `target.json`. + +Invalid trigger queries may fail before DataFusion execution. Such results are +observations from an analytics-configured, route-attested environment, not +claims that DataFusion executed the invalid query. The canary proves that each +fixture is capable of analytics execution. A static +`cluster.pluggable.dataformat=composite` startup setting is also required so +query-initial parse failures see the same routing configuration as valid +queries. + +Attestation uses assertions, not JUnit assumptions. A missing plugin or legacy +explain plan fails the lane. + +## 10. CI Workflow + +### 10.1 Final required topology + +```text +Get-CI-Image-Tag + |-------------------------------| + v v +standard-backend-validation analytics-backend-validation + | | + |---- standard artifacts |---- analytics artifacts + \ / + v v + detector-validation + (one OSD checkout/bootstrap, + two backend comparisons) + | + v + validation-result +``` + +The backend jobs run in parallel. The analytics job uses JDK 25 to match the +existing analytics compatibility workflow; the standard job keeps its current +JDK. + +Artifacts use distinct names and directories: + +```text +ppl-lint-backend-standard/ +ppl-lint-backend-analytics/ +ppl-lint-backend-standard-logs/ +ppl-lint-backend-analytics-logs/ +``` + +Before linting, `detector-validation` verifies: + +- Both target manifests exist. +- Both backend reports are non-empty. +- Both targets report the expected execution backend. +- Both targets report the same engine version and grammar hash. +- Analytics route attestation is complete. +- Every requested report exists, is non-empty, contains no duplicate identities, + and agrees with its target's execution backend. + +It then invokes `run-frontend-contract.mjs` twice against the same OSD checkout +and runtime grammar: + +```text +standard backend report -> detector-standard-report.json +analytics backend report -> detector-analytics-report.json +``` + +The duplicate detector pass costs seconds; the OSD bootstrap dominates the +job. Two explicit invocations are lower risk than redesigning the runner to +accept an arbitrary report collection. The result job also compares normalized +detector rows: rule/query identity, count, severity, and any asserted message +match. Equal counts alone are not sufficient parity. + +`validation-result` continues to use `if: always()` and becomes red unless all +three validation jobs succeeded. A skipped detector caused by either backend +failure therefore cannot appear green. + +### 10.2 Multi-version workflow + +The first analytics leg is `pr-build-analytics`. It is not added to every +released version: + +| Leg | Version | Grammar surface | Execution backend | +| --- | --- | --- | --- | +| Existing released legs | Released matrix | Runtime/compiled as configured | Standard | +| `pr-build` | Pull request build | Runtime bundle | Standard | +| `pr-build-analytics` | Pull request build | Runtime bundle | Analytics | + +`aggregate-versions.mjs` must understand the backend dimension before this leg +is added. It reports backend divergence separately and never turns an +analytics-only difference into version-scoping advice. + +The discovery corpus remains standard-only in the initial implementation. It +has no reviewed oracle and should not expand the analytics rollout's cost or +diagnostic surface. + +### 10.3 Local entry point + +`scripts/ppl-lint-rule-validation.sh` will gain an opt-in analytics mode, for +example `RUN_ANALYTICS=1`. It will support the existing local ZIP override +properties. Standard local behavior remains unchanged. + +## 11. Artifact Provenance + +The current Gradle default uses a mutable +`feature-datafusion/latest/linux/x64` URL. This is acceptable for early +observation but not for a required check. + +Before promotion: + +1. Add a checked-in compatibility lock describing the immutable analytics + feature build for the current OpenSearch line. +2. Add a Gradle property such as `analyticsFeatureBuildBase` so CI can pass the + immutable base while local development can retain the current default. +3. Verify SHA-256 for every downloaded plugin ZIP before cluster startup. +4. Record the immutable source, build ID, component versions, and hashes in + `target.json`. +5. Fail if installed plugin versions do not match the locked tuple. + +If an immutable artifact source cannot be provided, the analytics lane remains +non-enforcing. + +## 12. Failure Semantics + +| Failure | Classification | CI behavior | +| --- | --- | --- | +| Plugin download or checksum failure | Infrastructure | Retry download at most three times, then fail lane | +| Cluster does not become healthy | Infrastructure | Fail and upload cluster logs/thread dump | +| Required plugin absent or wrong version | Infrastructure | Fail before contracts | +| Fixture is not composite/Parquet | Route attestation | Fail before contracts | +| Explain/profile canary uses standard route | Route attestation | Fail before contracts | +| Standard and analytics grammar hashes differ | Candidate identity | Fail detector job | +| Missing/empty backend or detector report | Incomplete run | Fail; never aggregate survivors only | +| Missing analytics expectation | Coverage hole | Fail once analytics enforcement is enabled | +| Contract query transport timeout | Inconclusive run | Fail; never treat as backend acceptance | +| Trigger/control behavior differs from oracle | Semantic drift | Report backend, query, observed status/type, and remediation | +| Standard and analytics behavior differ | Execution-backend divergence | Report separately; do not suggest version scoping | +| Detector output differs between backend passes | Harness/context defect | Fail detector job | + +Semantic assertions are never retried. A retry could hide a nondeterministic +backend or detector defect. + +## 13. Diagnostics and Resource Bounds + +The analytics job will use: + +- A 30-minute GitHub job timeout. +- A bounded OpenSearch heap consistent with current workflows. +- One Netty direct arena and the existing native-access flags. +- One primary shard for required contract coverage. +- No credentials or fork secrets. +- `permissions: contents: read`. + +Always upload on failure: + +- `target.json` and analytics stack identity. +- Backend and detector reports. +- JUnit XML and HTML reports. +- Gradle test reports. +- Installed plugin list. +- Effective cluster and fixture index settings. +- Fixture mapping hashes and any fields stripped by the analytics fixture + helper. +- OpenSearch and test-cluster logs. +- Detector logs. +- Thread dumps for startup or query timeout. + +Reports must distinguish `accepted`, `rejected`, `error`, and +`not-applicable`. An absent `rejected` field is not equivalent to acceptance. + +## 14. Test Plan + +### 14.1 Harness unit tests + +Add Node tests for: + +- Schema version 3 compatibility and schema version 4 backend selection. +- Unknown or missing execution backend. +- Missing analytics oracle. +- Duplicate target identities. +- Same version with standard and analytics legs. +- Standard/analytics grammar mismatch. +- Backend transport error not being read as acceptance. +- Analytics divergence producing backend remediation, not version scoping. +- Detector parity between standard and analytics passes. +- Non-applicable handling and required-rule coverage holes. +- Summary and annotation output naming the execution backend. + +### 14.2 Java integration coverage + +Verify: + +- The standard `PplLintRuleValidationIT` behavior is unchanged. +- The analytics task installs the full stack. +- ACCOUNT and FLAT_OBJECT fixtures are composite/Parquet or fail explicitly. +- Explain and profile canaries attest the analytics route. +- Every scheduled contract emits one backend result per expected query. +- Report entries include `executionBackend`. +- A forced missing-plugin or standard-route configuration fails attestation. + +### 14.3 Workflow validation + +Use `workflow_dispatch` to validate: + +- Canonical OSD `main`. +- An explicit OSD branch/SHA. +- A successful dual-backend run. +- An intentionally wrong analytics oracle. +- An intentionally missing analytics artifact. +- A backend failure that skips detector work but still makes the final result + red. + +No production branch-protection change is made during this validation. + +## 15. Rollout + +### Phase 1: Identity and observation + +- Add execution-backend identity to targets and reports. +- Add the schema version 4 reader with version 3 compatibility. +- Make artifact consumers fail closed on missing, malformed, duplicate, or + conflicting identities. +- Add backend-aware aggregation and divergence remediation before introducing + an analytics leg. +- Add the managed analytics Gradle task and route attestation. +- Add `pr-build-analytics` to the non-required multi-version workflow. +- Missing analytics oracles are recorded as unscored coverage gaps during + observation. Infrastructure, identity, completeness, and attestation failures + remain red. Do not use `continue-on-error` inside the producer lane. + +### Phase 2: Baseline and review + +- Capture real analytics observations for the full contract corpus. +- Add reviewed analytics oracles. +- Resolve every default-error non-applicable case. +- Pin immutable analytics artifacts and verify their checksums. +- Pin the DataFusion-specific profile execution marker. +- Measure runtime and infrastructure reliability. + +Promotion requires: + +- Every scheduled contract has a reviewed analytics oracle. +- No `defaultError` contract is non-applicable. +- No unexplained semantic divergence remains. +- At least 25 consecutive green observation runs. +- At least 50 total runs with less than 1% infrastructure failure. +- Analytics job p95 runtime is at most 15 minutes. +- Artifact provenance is immutable and recorded. + +### Phase 3: Required check + +- Add `analytics-backend-validation` to the required single-version workflow. +- Make detector validation require both backend artifacts. +- Make `validation-result` require standard backend, analytics backend, and + detector success. +- Update the run manifest and PR summary to show both routes. + +There is no silent repository-variable bypass after promotion. An emergency +rollback requires an explicit workflow/branch-protection change and a tracking +issue. + +### Phase 4: Optional expansion + +After the required lane is stable, evaluate: + +- Matching released analytics stacks. +- A scheduled three-shard analytics leg. +- Analytics execution for the discovery corpus. +- Consolidating or retiring redundant parts of + `analytics-engine-compat.yml`. + +These are separate changes and are not prerequisites for initial enforcement. + +## 16. Planned File Changes + +| File | Change | +| --- | --- | +| `integ-test/build.gradle` | Add the full-stack analytics lint cluster/task and artifact lock inputs | +| `PplLintRuleValidationIT.java` | Select backend-specific oracles, attest route, and emit backend identity | +| `integ-test/src/test/resources/ppl-lint/contracts/*.spec.json` | Migrate to schema version 4 and add analytics oracles | +| `integ-test/src/test/resources/ppl-lint/contracts/manifest.json` | Bump schema metadata and document analytics coverage | +| `scripts/ppl-lint/run-frontend-contract.mjs` | Select the active backend oracle and emit backend identity | +| `scripts/ppl-lint/contract-schema.mjs` | Share strict Node schema, identity, and backend-oracle selection | +| `scripts/ppl-lint/aggregate-versions.mjs` | Key and render legs by execution backend | +| `scripts/ppl-lint/drift.mjs` | Add execution-backend divergence and remediation | +| `scripts/ppl-lint/annotate.mjs` | Attach backend-specific findings to contract declarations | +| `scripts/ppl-lint/assemble-run-manifest.mjs` | Record both targets and job results | +| `scripts/ppl-lint/__tests__/*` | Cover schema, identity, aggregation, and remediation changes | +| `.github/workflows/ppl-lint-multiversion-validation.yml` | Add the observation leg | +| `.github/workflows/ppl-lint-rule-validation.yml` | Add the required lane after promotion | +| `scripts/ppl-lint-rule-validation.sh` | Add opt-in local analytics reproduction | +| `scripts/ppl-lint/README.md` | Document backend-aware contracts and commands | +| Analytics compatibility lock (path TBD) | Pin immutable plugin URLs, versions, and SHA-256 values before required promotion | + +## 17. Success Criteria + +The work is complete when: + +1. A pull request can produce standard and analytics observations from the same + SQL commit and grammar. +2. CI proves the analytics route instead of relying on a configuration flag. +3. Every scheduled contract has an explicit analytics result. +4. Reports cannot confuse backend divergence with version drift. +5. Missing analytics coverage cannot pass as agreement. +6. The required result fails when either backend or the OSD detector contract + fails. +7. A failed run includes enough immutable identity and logs to reproduce the + target that was tested. + +## 18. Open Questions + +1. Which system owns publishing and retaining immutable analytics feature-build + tuples for required CI? +2. Should the artifact compatibility lock live in this repository or be + generated by the OpenSearch feature-build pipeline? +3. Which current contract queries produce intentional analytics behavior + differences once the first observation run is available? +4. Will OSD eventually expose a reliable execution-backend signal to lint + context? If so, detector expectations may later become backend-aware. +5. After the full semantic lane is required, does the smaller coexistence smoke + workflow still provide enough independent value to keep? diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 5dffd44560e..ee208050838 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -294,9 +294,12 @@ def getGeoSpatialPlugin() { } } -// fetch from the feature-build artifact for now (linux/x64 only; for local dev pass -PanalyticsEngineZip=/path instead). +// Fetch from the mutable feature-build artifact for observation (linux/x64 only). CI can +// select a specific build with -PanalyticsFeatureBuildBase, and local development can pass +// individual plugin ZIP overrides such as -PanalyticsEngineZip=/path. ext.pluginVersion = opensearch_version.tokenize('-')[0] -ext.featureBuildBase = "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildBase = project.findProperty('analyticsFeatureBuildBase') ?: + "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip" ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip" ext.arrowBaseZipDest = "${buildDir}/distributions/arrow-base-${pluginVersion}-SNAPSHOT.zip" @@ -452,6 +455,35 @@ testClusters { // Composite-default cluster: PPL queries route to the analytics engine unless excluded. setting 'cluster.pluggable.dataformat', 'composite' } + analyticsEnginePplLintIT { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getArrowBasePlugin()) + plugin(getArrowFlightRpcPlugin()) + plugin(getAnalyticsEnginePlugin()) + plugin(getCompositeEnginePlugin()) + plugin(getParquetDataFormatPlugin()) + plugin(getAnalyticsBackendLucenePlugin()) + plugin(getAnalyticsBackendDatafusionPlugin()) + plugin ":opensearch-sql-plugin" + setting 'cluster.pluggable.dataformat.enabled', 'true' + setting 'cluster.pluggable.dataformat', 'composite' + setting 'cluster.composite.primary_data_format', 'parquet' + setting 'cluster.composite.secondary_data_formats', '[lucene]' + // Arrow Flight / streaming transport requirements + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' + systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' + // Native library path for DataFusion/parquet -- pass via -PnativeLibPath=/path/to/release/ + if (project.findProperty('nativeLibPath')) { + systemProperty 'java.library.path', project.findProperty('nativeLibPath') + } + } } def isPrometheusRunning() { @@ -511,6 +543,24 @@ task analyticsEngineCompatIT(type: RestIntegTestTask) { } } +task analyticsEnginePplLintIT(type: RestIntegTestTask) { + useCluster testClusters.analyticsEnginePplLintIT + dependsOn downloadArrowBaseZip, downloadArrowFlightRpcZip, downloadAnalyticsEngineZip, + downloadCompositeEngineZip, downloadParquetDataFormatZip, + downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip + dependsOn ':opensearch-sql-plugin:bundlePlugin' + + systemProperty 'tests.analytics.parquet_indices', 'true' + systemProperty 'tests.analytics.num_shards', '1' + systemProperty 'ppl.lint.execution_backend', 'analytics' + systemProperty 'ppl.lint.analytics.stack.source', featureBuildBase + systemProperty 'tests.security.manager', 'false' + + filter { + includeTestsMatching 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' + } +} + task analyticsEngineSecurityIT(type: RestIntegTestTask) { dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip, downloadArrowBaseZip, downloadAnalyticsBackendLuceneZip, downloadParquetDataFormatZip, downloadCompositeEngineZip, downloadAnalyticsBackendDatafusionZip dependsOn ':opensearch-sql-plugin:bundlePlugin' diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 7ed5e6ad989..01012b7406d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -12,7 +12,10 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -29,14 +32,14 @@ import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Backend half of the schema-v3 PPL lint rule validation contract. + * Backend half of the schema-v3/schema-v4 PPL lint rule validation contract. * *

    This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from * the current checkout. For every contract (see {@code * src/test/resources/ppl-lint/contracts/*.spec.json}) it selects the single {@code expectations[]} * entry that matches the candidate backend version (exactly one must match, or the contract fails - * before any query runs), applies the contract's cluster settings, and asserts, per query's {@code - * backend.kind}: + * before any query runs), applies the contract's cluster settings, and asserts the oracle selected + * for {@code ppl.lint.execution_backend}: * *

      *
    • {@code rejection} — the query returns the contracted HTTP status and structured error body @@ -77,10 +80,27 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private static final String CONTRACT_DIR = "src/test/resources/ppl-lint/contracts"; private static final String MANIFEST = CONTRACT_DIR + "/manifest.json"; private static final String GRAMMAR_API_ENDPOINT = "/_plugins/_ppl/_grammar"; + private static final String EXECUTION_BACKEND_PROPERTY = "ppl.lint.execution_backend"; + private static final String ANALYTICS_SHARD_COUNT_PROPERTY = "tests.analytics.num_shards"; + private static final String[] REQUIRED_ANALYTICS_PLUGIN_COMPONENTS = { + "job-scheduler", + "arrow-base", + "arrow-flight-rpc", + "analytics-engine", + "analytics-backend-lucene", + "analytics-backend-datafusion", + "parquet-data-format", + "composite-engine", + "opensearch-sql" + }; /** Which contracts to run this session; PR is the fast blocking subset. */ private final String schedule = System.getProperty("ppl.lint.schedule", "pr"); + /** Execution route whose backend oracle and artifact identity this run represents. */ + private final ExecutionBackend executionBackend = + ExecutionBackend.parse(System.getProperty(EXECUTION_BACKEND_PROPERTY, "standard")); + /** * Observe-only mode, used by the multi-version workflow ({@code * .github/workflows/ppl-lint-multiversion-validation.yml}). @@ -101,6 +121,13 @@ public class PplLintRuleValidationIT extends PPLIntegTestCase { private int[] clusterVersion; private String engineVersionRaw; + private final JSONObject analyticsRouteAttestation = + new JSONObject() + .put("pluginsVerified", false) + .put("clusterSettingsVerified", false) + .put("fixtureIndicesVerified", false) + .put("explainVerified", false) + .put("profiledExecutionVerified", false); /** * Whether this cluster recognizes the Calcite settings at all. False on a pre-Calcite (2.x) @@ -175,18 +202,32 @@ public void testValidatesLintRuleContracts() throws IOException { List contracts = loadScheduledContracts(); List failures = new ArrayList<>(); JSONArray report = new JSONArray(); + if (contracts.isEmpty()) { + failures.add("[contracts] no contracts were selected for schedule \"" + schedule + "\""); + } + + boolean routeAttested = + executionBackend != ExecutionBackend.ANALYTICS || attestAnalyticsRoute(failures); // Export the candidate grammar bundle + target manifest while the cluster is // alive. Runs before the contract loop so the artifacts are emitted even if a // contract later fails. exportGrammarArtifacts(failures); - for (JSONObject contract : contracts) { - String ruleId = contract.getString("ruleId"); - runContract(contract, ruleId, failures, report); + // A failed route attestation is infrastructure failure, not backend behavior. + // Do not score any contract against a route that was not proven. + if (routeAttested) { + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + runContract(contract, ruleId, failures, report); + } } - writeReport(report); + try { + writeReport(report); + } catch (IOException e) { + failures.add("[report] failed to write backend report: " + e.getMessage()); + } if (!failures.isEmpty()) { fail( @@ -200,12 +241,53 @@ public void testValidatesLintRuleContracts() throws IOException { private void runContract( JSONObject contract, String ruleId, List failures, JSONArray report) throws IOException { + int schemaVersion = contract.getInt("schemaVersion"); + if (schemaVersion != 3 && schemaVersion != 4) { + failures.add( + "[" + ruleId + "] unsupported schemaVersion " + schemaVersion + " (expected 3 or 4)"); + return; + } + String index = contract.getString("index"); JSONObject queries = contract.getJSONObject("queries"); JSONArray expectations = contract.getJSONArray("expectations"); JSONObject fixture = contract.optJSONObject("backendFixture"); boolean calciteOn = fixtureCalciteEnabled(fixture); + if (expectations.length() == 0) { + failures.add("[" + ruleId + "] expectations must not be empty"); + return; + } + + if (!validateAllExpectations(ruleId, queries, expectations, schemaVersion, failures)) { + return; + } + + List matches = matchingExpectations(expectations, calciteOn); + if (matches.size() > 1) { + failures.add( + "[" + + ruleId + + "] " + + matches.size() + + " expectations match backend version " + + backendVersionLabel() + + " (exactly one required)"); + return; + } + + JSONObject selected = matches.isEmpty() ? null : matches.get(0); + if (selected == null && !observeOnly) { + failures.add( + "[" + + ruleId + + "] no version expectation matches backend version " + + backendVersionLabel()); + return; + } + + JSONObject expectedQueries = selected == null ? null : selected.getJSONObject("queries"); + // A contract whose fixture index never got created cannot produce a meaningful // observation: every query would fail with IndexNotFoundException regardless of // the rule. Report each case as an error so the aggregator counts it as @@ -216,48 +298,54 @@ private void runContract( return; } + if (!observeOnly + && recordEnforcementCoverageGaps( + ruleId, index, queries, expectedQueries, schemaVersion, failures, report)) { + return; + } + List applied = applyClusterSettings(fixture); try { - // In observe-only mode, "no expectation matches this version" is information, - // not a failure — a rule the corpus does not pin for THIS engine is exactly - // what the multi-version matrix is here to learn. selectExpectation records - // into whatever list it is handed, so hand it a scratch list we discard; - // otherwise the leg both records the observation AND fails, which is what - // kept union-min-datasets (a >=3.7 rule) failing the 3.6 leg. - List selectionFailures = observeOnly ? new ArrayList<>() : failures; - JSONObject selected = selectExpectation(ruleId, expectations, calciteOn, selectionFailures); if (selected == null) { - if (!observeOnly) { - return; // no/ambiguous version expectation — failure already recorded. - } // Record the raw behavior of every query and let the aggregator decide // whether the gap matters (out-of-scope rule vs a real coverage hole). - observeAllQueries(ruleId, index, queries, report); + observeAllQueries(ruleId, index, queries, failures, report); return; } - JSONObject expectedQueries = selected.getJSONObject("queries"); - for (String queryName : expectedQueries.keySet()) { - if (!queries.has(queryName)) { - failures.add( - "[" - + ruleId - + "] expectation references unknown query \"" - + queryName - + "\" (not in the top-level queries map)"); - continue; - } + + for (String queryName : queries.keySet()) { JSONObject queryDef = queries.getJSONObject(queryName); String role = queryDef.optString("role", "trigger"); String query = queryDef.getString("query").replace("{{index}}", index); JSONObject expected = expectedQueries.getJSONObject(queryName); - JSONObject backend = expected.getJSONObject("backend"); + JSONObject backend = resolveBackendOracle(schemaVersion, expected); + if (backend == null) { + recordMissingOracle(ruleId, queryName, role, query, schemaVersion, failures, report); + continue; + } + String kind = backend.getString("kind"); JSONObject entry = reportEntry(ruleId, queryName, role, query, kind); + if ("not-applicable".equals(kind)) { + recordNotApplicable(ruleId, queryName, backend, entry, report); + continue; + } + try { verifyCase(kind, queryName, query, backend, entry); entry.put("outcome", "pass"); log(ruleId, queryName, "PASS (" + kind + ", " + role + ")"); + } catch (IOException e) { + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend query transport failed: " + + String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR (" + kind + "): " + e.getMessage()); } catch (AssertionError | RuntimeException e) { entry.put("outcome", observeOnly ? "observed-mismatch" : "fail"); entry.put("error", String.valueOf(e.getMessage())); @@ -327,7 +415,7 @@ private void recordUnusableContract( * a blank row that would read as agreement. */ private void observeAllQueries( - String ruleId, String index, JSONObject queries, JSONArray report) { + String ruleId, String index, JSONObject queries, List failures, JSONArray report) { for (String queryName : queries.keySet()) { JSONObject queryDef = queries.getJSONObject(queryName); String role = queryDef.optString("role", "trigger"); @@ -344,6 +432,13 @@ private void observeAllQueries( // A transport-level problem is a broken run, not an engine verdict; mark it // so the aggregator does not read the absence of a rejection as acceptance. entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend observation failed: " + + String.valueOf(e.getMessage())); log(ruleId, queryName, "ERROR: " + e.getMessage()); } report.put(entry); @@ -351,42 +446,412 @@ private void observeAllQueries( } /** - * Select the single {@code expectations[]} entry that applies to the candidate backend version - * and engine. Exactly one must match: zero means the rule test does not cover this version - * (design §9), and more than one means overlapping ranges — both fail before execution (§5.3). + * A missing backend oracle is a coverage result, not an invitation to borrow another backend's + * expectation. Observation executes the query exactly once and records its raw behavior; + * enforcement records the gap without executing or scoring the query. */ - private JSONObject selectExpectation( - String ruleId, JSONArray expectations, boolean calciteOn, List failures) { - List matches = new ArrayList<>(); + private void recordMissingOracle( + String ruleId, + String queryName, + String role, + String query, + int schemaVersion, + List failures, + JSONArray report) { + String reason = + schemaVersion == 3 + ? "schema v3 provides only a standard backend oracle" + : "schema v4 has no " + executionBackend.id + " entry in expected query backends"; + JSONObject entry = + reportEntry(ruleId, queryName, role, query, "coverage-missing") + .put("coverage", "missing") + .put("reason", reason); + + if (!observeOnly) { + entry.put("outcome", "coverage-missing"); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] missing " + + executionBackend.id + + " backend oracle: " + + reason); + report.put(entry); + log(ruleId, queryName, "COVERAGE MISSING (" + executionBackend.id + ")"); + return; + } + + try { + BackendObservation obs = observeBackend(query); + entry + .put("rejected", obs.rejected) + .put("observed", obs.toJson()) + .put("outcome", "coverage-missing"); + log( + ruleId, + queryName, + "COVERAGE MISSING; OBSERVED (" + (obs.rejected ? "rejected" : "accepted") + ")"); + } catch (IOException | RuntimeException e) { + entry.put("outcome", "error").put("error", String.valueOf(e.getMessage())); + failures.add( + "[" + + ruleId + + "/" + + queryName + + "] backend observation failed: " + + String.valueOf(e.getMessage())); + log(ruleId, queryName, "ERROR: " + e.getMessage()); + } + report.put(entry); + } + + /** + * Enforcement must establish complete backend-oracle coverage before executing any query in the + * contract. This avoids producing partially scored evidence when a later query has no oracle. + */ + private boolean recordEnforcementCoverageGaps( + String ruleId, + String index, + JSONObject queries, + JSONObject expectedQueries, + int schemaVersion, + List failures, + JSONArray report) { + boolean missing = false; + for (String queryName : queries.keySet()) { + JSONObject expected = expectedQueries.getJSONObject(queryName); + if (resolveBackendOracle(schemaVersion, expected) != null) { + continue; + } + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + recordMissingOracle(ruleId, queryName, role, query, schemaVersion, failures, report); + missing = true; + } + return missing; + } + + /** Record an explicit schema-v4 non-applicable oracle without executing the query. */ + private void recordNotApplicable( + String ruleId, String queryName, JSONObject backend, JSONObject entry, JSONArray report) { + String reason = backend.getString("reason"); + entry + .put("outcome", "not-applicable") + .put("reason", reason) + .put("owner", backend.getString("owner")) + .put("issue", backend.getString("issue")); + report.put(entry); + log(ruleId, queryName, "NOT APPLICABLE (" + executionBackend.id + ")"); + } + + /** + * Resolve the execution backend oracle without fallback. Schema v3 is standard-only; schema v4 + * requires an explicit entry in {@code backends}. + */ + private JSONObject resolveBackendOracle(int schemaVersion, JSONObject expected) { + if (schemaVersion == 3) { + return executionBackend == ExecutionBackend.STANDARD + ? expected.getJSONObject("backend") + : null; + } + if (schemaVersion == 4) { + JSONObject backends = expected.optJSONObject("backends"); + return backends != null && backends.has(executionBackend.id) + ? backends.getJSONObject(executionBackend.id) + : null; + } + throw new IllegalArgumentException("unsupported contract schemaVersion " + schemaVersion); + } + + /** + * Validate every expectation before version selection or query execution. Observation mode may + * tolerate a missing route oracle, but it must never turn a malformed oracle into observed drift. + */ + private boolean validateAllExpectations( + String ruleId, + JSONObject declaredQueries, + JSONArray expectations, + int schemaVersion, + List failures) { + Set declared = new LinkedHashSet<>(declaredQueries.keySet()); + boolean valid = true; + if (declared.isEmpty()) { + failures.add("[" + ruleId + "] queries must not be empty"); + valid = false; + } for (int i = 0; i < expectations.length(); i++) { - JSONObject exp = expectations.getJSONObject(i); - if (!versionMatchesRange(exp.optString("version", null))) { + String expectationPath = "expectations[" + i + "]"; + Object expectationValue = expectations.opt(i); + if (!(expectationValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + expectationPath + " must be an object"); + valid = false; continue; } - String engine = exp.optString("engine", ""); - if ("calcite".equals(engine) && !calciteOn) { + JSONObject expectation = (JSONObject) expectationValue; + Object expectationQueriesValue = expectation.opt("queries"); + if (!(expectationQueriesValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + expectationPath + ".queries must be an object"); + valid = false; continue; } - matches.add(exp); + JSONObject expectationQueries = (JSONObject) expectationQueriesValue; + Set expected = new LinkedHashSet<>(expectationQueries.keySet()); + if (!declared.equals(expected)) { + Set missingFromExpectation = new LinkedHashSet<>(declared); + missingFromExpectation.removeAll(expected); + Set unknownInExpectation = new LinkedHashSet<>(expected); + unknownInExpectation.removeAll(declared); + failures.add( + "[" + + ruleId + + "] " + + expectationPath + + " query keys must exactly match top-level queries" + + "; missing from expectation=" + + missingFromExpectation + + "; unknown in expectation=" + + unknownInExpectation); + valid = false; + } + + for (String queryName : expected) { + String queryPath = expectationPath + ".queries." + queryName; + Object queryExpectationValue = expectationQueries.opt(queryName); + if (!(queryExpectationValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + queryPath + " must be an object"); + valid = false; + continue; + } + JSONObject queryExpectation = (JSONObject) queryExpectationValue; + if (schemaVersion == 3) { + Object backendValue = queryExpectation.opt("backend"); + if (!(backendValue instanceof JSONObject)) { + failures.add( + "[" + ruleId + "] " + queryPath + ".backend must be a schema-v3 oracle object"); + valid = false; + continue; + } + valid &= + validateBackendOracle( + ruleId, queryPath + ".backend", (JSONObject) backendValue, failures); + continue; + } + + if (!queryExpectation.has("backends")) { + continue; + } + Object backendsValue = queryExpectation.opt("backends"); + if (!(backendsValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + queryPath + ".backends must be an object"); + valid = false; + continue; + } + JSONObject backends = (JSONObject) backendsValue; + for (String backend : backends.keySet()) { + String backendPath = queryPath + ".backends." + backend; + if (!"standard".equals(backend) && !"analytics".equals(backend)) { + failures.add( + "[" + + ruleId + + "] " + + queryPath + + " declares unknown execution backend \"" + + backend + + "\""); + valid = false; + continue; + } + Object oracleValue = backends.opt(backend); + if (!(oracleValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + backendPath + " must be an oracle object"); + valid = false; + continue; + } + valid &= validateBackendOracle(ruleId, backendPath, (JSONObject) oracleValue, failures); + } + } + } + return valid; + } + + private boolean validateBackendOracle( + String ruleId, String path, JSONObject oracle, List failures) { + int initialFailureCount = failures.size(); + String kind = requireNonBlankString(ruleId, path + ".kind", oracle.opt("kind"), failures); + if (kind == null) { + return false; } - String versionLabel = engineVersionRaw == null ? "unknown" : engineVersionRaw; - if (matches.size() == 1) { - return matches.get(0); + + if ("not-applicable".equals(kind)) { + requireNonBlankString(ruleId, path + ".reason", oracle.opt("reason"), failures); + requireNonBlankString(ruleId, path + ".owner", oracle.opt("owner"), failures); + requireNonBlankString(ruleId, path + ".issue", oracle.opt("issue"), failures); + return failures.size() == initialFailureCount; } - if (matches.isEmpty()) { - failures.add( - "[" + ruleId + "] no version expectation matches backend version " + versionLabel); - } else { + + Integer httpStatus = + requireInteger(ruleId, path + ".httpStatus", oracle.opt("httpStatus"), 100, 599, failures); + switch (kind) { + case "rejection": + validateRejectionOracle(ruleId, path, oracle, httpStatus, failures); + break; + case "result-shape": + requireHttpOk(ruleId, path, httpStatus, failures); + validateResultShapeOracle(ruleId, path, oracle, failures); + break; + case "advisory": + requireHttpOk(ruleId, path, httpStatus, failures); + validateAdvisoryOracle(ruleId, path, oracle, failures); + break; + default: + failures.add("[" + ruleId + "] " + path + ".kind is unknown: \"" + kind + "\""); + break; + } + return failures.size() == initialFailureCount; + } + + private void validateRejectionOracle( + String ruleId, String path, JSONObject oracle, Integer httpStatus, List failures) { + Object bodyValue = oracle.opt("body"); + if (!(bodyValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".body must be an object"); + return; + } + JSONObject body = (JSONObject) bodyValue; + Integer bodyStatus = + requireInteger(ruleId, path + ".body.status", body.opt("status"), 100, 599, failures); + if (httpStatus != null && bodyStatus != null && !httpStatus.equals(bodyStatus)) { + failures.add("[" + ruleId + "] " + path + ".httpStatus must equal " + path + ".body.status"); + } + + if (!body.has("error")) { + return; + } + Object errorValue = body.opt("error"); + if (!(errorValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".body.error must be an object"); + return; + } + JSONObject error = (JSONObject) errorValue; + if (error.has("type")) { + requireNonBlankString(ruleId, path + ".body.error.type", error.opt("type"), failures); + } + if (error.has("reason")) { + requireNonBlankString(ruleId, path + ".body.error.reason", error.opt("reason"), failures); + } + } + + private void validateResultShapeOracle( + String ruleId, String path, JSONObject oracle, List failures) { + if (!oracle.has("expect")) { + return; + } + Object expectValue = oracle.opt("expect"); + if (!(expectValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".expect must be an object"); + return; + } + JSONObject expect = (JSONObject) expectValue; + if (expect.has("datarowsNonEmpty") && !(expect.opt("datarowsNonEmpty") instanceof Boolean)) { + failures.add("[" + ruleId + "] " + path + ".expect.datarowsNonEmpty must be a boolean"); + } + if (expect.has("datarowsCount")) { + requireInteger( + ruleId, + path + ".expect.datarowsCount", + expect.opt("datarowsCount"), + 0, + Integer.MAX_VALUE, + failures); + } + if (expect.has("columnAllNull")) { + requireNonBlankString( + ruleId, path + ".expect.columnAllNull", expect.opt("columnAllNull"), failures); + } + } + + private void validateAdvisoryOracle( + String ruleId, String path, JSONObject oracle, List failures) { + if (!oracle.has("expect")) { + return; + } + Object expectValue = oracle.opt("expect"); + if (!(expectValue instanceof JSONObject)) { + failures.add("[" + ruleId + "] " + path + ".expect must be an object"); + return; + } + JSONObject expect = (JSONObject) expectValue; + if (expect.has("accepted") && !Boolean.TRUE.equals(expect.opt("accepted"))) { + failures.add("[" + ruleId + "] " + path + ".expect.accepted must be true"); + } + } + + private void requireHttpOk( + String ruleId, String path, Integer httpStatus, List failures) { + if (httpStatus != null && httpStatus != 200) { + failures.add("[" + ruleId + "] " + path + ".httpStatus must be 200"); + } + } + + private String requireNonBlankString( + String ruleId, String path, Object value, List failures) { + if (!(value instanceof String) || ((String) value).trim().isEmpty()) { + failures.add("[" + ruleId + "] " + path + " must be a non-blank string"); + return null; + } + return (String) value; + } + + private Integer requireInteger( + String ruleId, String path, Object value, int minimum, int maximum, List failures) { + if (!(value instanceof Number)) { + failures.add("[" + ruleId + "] " + path + " must be an integer"); + return null; + } + double numeric = ((Number) value).doubleValue(); + if (!Double.isFinite(numeric) + || numeric != Math.rint(numeric) + || numeric < minimum + || numeric > maximum) { failures.add( "[" + ruleId + "] " - + matches.size() - + " expectations match backend version " - + versionLabel - + " (exactly one required)"); + + path + + " must be an integer from " + + minimum + + " through " + + maximum); + return null; } - return null; + return ((Number) value).intValue(); + } + + /** + * Find the expectations that apply to the candidate version and planner. The caller treats zero + * matches as raw-observation-only and multiple matches as fatal in every mode. + */ + private List matchingExpectations(JSONArray expectations, boolean calciteOn) { + List matches = new ArrayList<>(); + for (int i = 0; i < expectations.length(); i++) { + JSONObject exp = expectations.getJSONObject(i); + if (!versionMatchesRange(exp.optString("version", null))) { + continue; + } + String engine = exp.optString("engine", ""); + if ("calcite".equals(engine) && !calciteOn) { + continue; + } + matches.add(exp); + } + return matches; + } + + private String backendVersionLabel() { + return engineVersionRaw == null ? "unknown" : engineVersionRaw; } private void verifyCase( @@ -428,8 +893,7 @@ private BackendObservation observeBackend(String query) throws IOException { try { body = new JSONObject(getResponseBody(e.getResponse(), true)); } catch (IOException ioe) { - throw new RuntimeException( - "failed to read rejection response body for query: " + query, ioe); + throw new IOException("failed to read rejection response body for query: " + query, ioe); } return BackendObservation.rejected(status, body); } @@ -624,15 +1088,403 @@ static BackendObservation rejected(int status, JSONObject body) { JSONObject toJson() { JSONObject o = new JSONObject().put("httpStatus", status).put("rejected", rejected); if (body != null) { + o.put("body", body); JSONObject err = body.optJSONObject("error"); if (err != null) { o.put("type", err.opt("type")).put("reason", err.opt("reason")); } } + if (response != null) { + o.put("response", response); + } return o; } } + // --- analytics route attestation ------------------------------------------ + + /** + * Prove the analytics route before any contract is scored. Each check is retained in the target + * manifest, including failures, so a missing route cannot be mistaken for backend coverage. + */ + private boolean attestAnalyticsRoute(List failures) { + boolean plugins = + runAnalyticsAttestationCheck( + "pluginsVerified", "required plugins", this::verifyAnalyticsPlugins, failures); + boolean clusterSettings = + runAnalyticsAttestationCheck( + "clusterSettingsVerified", + "cluster settings", + this::verifyAnalyticsClusterSettings, + failures); + boolean fixtureIndices = + runAnalyticsAttestationCheck( + "fixtureIndicesVerified", + "fixture index settings", + this::verifyAnalyticsFixtureIndices, + failures); + boolean explain = + runAnalyticsAttestationCheck( + "explainVerified", "explain route", this::verifyAnalyticsExplainCanaries, failures); + boolean profile = + runAnalyticsAttestationCheck( + "profiledExecutionVerified", + "profiled execution", + this::verifyAnalyticsProfileCanaries, + failures); + return plugins && clusterSettings && fixtureIndices && explain && profile; + } + + private boolean runAnalyticsAttestationCheck( + String targetField, String label, AttestationCheck check, List failures) { + try { + check.run(); + analyticsRouteAttestation.put(targetField, true); + log("route-attestation", label, "PASS"); + return true; + } catch (Exception | AssertionError e) { + analyticsRouteAttestation.put(targetField, false); + failures.add( + "[route-attestation/" + + label + + "] " + + (e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage())); + log("route-attestation", label, "FAIL: " + e.getMessage()); + return false; + } + } + + private void verifyAnalyticsPlugins() throws IOException { + Response response = + client() + .performRequest(new Request("GET", "/_cat/plugins?format=json&h=component,version")); + JSONArray plugins = new JSONArray(getResponseBody(response, true)); + List installed = new ArrayList<>(); + for (int i = 0; i < plugins.length(); i++) { + installed.add(plugins.getJSONObject(i).getString("component")); + } + analyticsRouteAttestation.put("plugins", plugins); + + requireAttestation( + engineVersionRaw != null && !engineVersionRaw.trim().isEmpty(), + "cluster engine version is unavailable for plugin compatibility checks"); + String expectedVersionPrefix = engineVersionRaw.split("-")[0]; + for (String required : REQUIRED_ANALYTICS_PLUGIN_COMPONENTS) { + JSONObject matched = null; + for (int i = 0; i < plugins.length(); i++) { + JSONObject plugin = plugins.getJSONObject(i); + if (pluginComponentMatches(plugin.getString("component"), required)) { + matched = plugin; + break; + } + } + requireAttestation( + matched != null, + "required plugin component matching \"" + + required + + "\" is missing; installed=" + + installed); + String version = matched.optString("version", ""); + requireAttestation( + version.equals(expectedVersionPrefix) + || version.startsWith(expectedVersionPrefix + ".") + || version.startsWith(expectedVersionPrefix + "-"), + "plugin " + + matched.getString("component") + + " version " + + version + + " is incompatible with engine " + + engineVersionRaw); + } + } + + private boolean pluginComponentMatches(String component, String required) { + return component.equals(required) || component.endsWith("-" + required); + } + + private void verifyAnalyticsClusterSettings() throws IOException { + Response nodesResponse = + client().performRequest(new Request("GET", "/_nodes/settings?flat_settings=true")); + JSONObject nodes = new JSONObject(getResponseBody(nodesResponse, true)).getJSONObject("nodes"); + requireAttestation(nodes.length() > 0, "node settings response contained no nodes"); + for (String nodeId : nodes.keySet()) { + String startupDataFormat = + nodes + .getJSONObject(nodeId) + .getJSONObject("settings") + .optString("cluster.pluggable.dataformat", ""); + String startupEnabled = + nodes + .getJSONObject(nodeId) + .getJSONObject("settings") + .optString("cluster.pluggable.dataformat.enabled", ""); + requireAttestation( + "composite".equals(startupDataFormat), + "node " + + nodeId + + " startup cluster.pluggable.dataformat must be composite but was \"" + + startupDataFormat + + "\""); + requireAttestation( + "true".equals(startupEnabled), + "node " + + nodeId + + " startup cluster.pluggable.dataformat.enabled must be true but was \"" + + startupEnabled + + "\""); + } + + Response response = + client() + .performRequest( + new Request("GET", "/_cluster/settings?flat_settings=true&include_defaults=true")); + JSONObject settings = new JSONObject(getResponseBody(response, true)); + + requireEffectiveSetting(settings, "cluster.pluggable.dataformat", "composite"); + requireEffectiveSetting(settings, "cluster.pluggable.dataformat.enabled", "true"); + requireEffectiveSetting(settings, "cluster.composite.primary_data_format", "parquet"); + requireEffectiveSettingContains(settings, "cluster.composite.secondary_data_formats", "lucene"); + analyticsRouteAttestation.put("clusterSettings", settings); + } + + private void verifyAnalyticsFixtureIndices() throws IOException { + int expectedShards = analyticsShardCount(); + JSONObject documentCounts = new JSONObject(); + JSONObject fixtureIndices = new JSONObject(); + analyticsRouteAttestation + .put("fixtureDocumentCounts", documentCounts) + .put("fixtureIndices", fixtureIndices); + for (String indexEnum : requiredIndexEnums()) { + String indexName = Index.valueOf(indexEnum).getName(); + Response response = + client() + .performRequest( + new Request( + "GET", + "/" + indexName + "/_settings?flat_settings=true&include_defaults=true")); + JSONObject body = new JSONObject(getResponseBody(response, true)); + JSONObject settings = body.getJSONObject(indexName).getJSONObject("settings"); + JSONObject fixtureEvidence = new JSONObject().put("settings", settings); + fixtureIndices.put(indexName, fixtureEvidence); + + Response mappingResponse = + client().performRequest(new Request("GET", "/" + indexName + "/_mapping")); + JSONObject mappingBody = new JSONObject(getResponseBody(mappingResponse, true)); + JSONObject mapping = mappingBody.getJSONObject(indexName).getJSONObject("mappings"); + fixtureEvidence.put("mappingHash", sha256(canonicalJson(mapping))).put("mapping", mapping); + + requireIndexSetting(indexName, settings, "index.pluggable.dataformat.enabled", "true"); + requireIndexSetting(indexName, settings, "index.pluggable.dataformat", "composite"); + requireIndexSetting(indexName, settings, "index.composite.primary_data_format", "parquet"); + requireIndexSettingContains( + indexName, settings, "index.composite.secondary_data_formats", "lucene"); + requireIndexSetting( + indexName, settings, "index.number_of_shards", Integer.toString(expectedShards)); + + Response countResponse = + client().performRequest(new Request("GET", "/" + indexName + "/_count")); + long count = new JSONObject(getResponseBody(countResponse, true)).getLong("count"); + requireAttestation( + count > 0, + "fixture " + indexName + " contains no documents; fixture ingestion did not complete"); + documentCounts.put(indexName, count); + fixtureEvidence.put("documentCount", count); + } + } + + private String canonicalJson(Object value) { + if (value == null || value == JSONObject.NULL) { + return "null"; + } + if (value instanceof JSONObject) { + JSONObject object = (JSONObject) value; + List keys = new ArrayList<>(object.keySet()); + Collections.sort(keys); + StringBuilder canonical = new StringBuilder("{"); + for (int i = 0; i < keys.size(); i++) { + if (i > 0) { + canonical.append(','); + } + String key = keys.get(i); + canonical.append(JSONObject.quote(key)).append(':').append(canonicalJson(object.get(key))); + } + return canonical.append('}').toString(); + } + if (value instanceof JSONArray) { + JSONArray array = (JSONArray) value; + StringBuilder canonical = new StringBuilder("["); + for (int i = 0; i < array.length(); i++) { + if (i > 0) { + canonical.append(','); + } + canonical.append(canonicalJson(array.get(i))); + } + return canonical.append(']').toString(); + } + if (value instanceof String) { + return JSONObject.quote((String) value); + } + if (value instanceof Number || value instanceof Boolean) { + return value.toString(); + } + throw new IllegalArgumentException( + "unsupported JSON value type in fixture mapping: " + value.getClass().getName()); + } + + private String sha256(String value) { + try { + byte[] digest = + MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder("sha256:"); + for (byte octet : digest) { + hex.append(String.format(Locale.ROOT, "%02x", octet & 0xff)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 digest is unavailable", e); + } + } + + private void verifyAnalyticsExplainCanaries() throws IOException { + for (String indexEnum : requiredIndexEnums()) { + String query = analyticsCanaryQuery(indexEnum); + String explained = explainQueryToString(query); + requireAttestation( + explained.contains("LogicalTableScan(table=[[opensearch,"), + "fixture " + indexEnum + " did not use LogicalTableScan(opensearch): " + explained); + requireAttestation( + !explained.contains("CalciteLogicalIndexScan"), + "fixture " + indexEnum + " fell back to CalciteLogicalIndexScan: " + explained); + } + } + + private void verifyAnalyticsProfileCanaries() throws IOException { + JSONArray executionTypes = new JSONArray(); + for (String indexEnum : requiredIndexEnums()) { + JSONObject response = runProfiledPplQuery(analyticsCanaryQuery(indexEnum)); + JSONObject profile = response.getJSONObject("profile"); + JSONArray stages = profile.getJSONObject("plan").getJSONArray("stages"); + requireAttestation( + stages.length() > 0, "fixture " + indexEnum + " profile returned no execution stages"); + for (int i = 0; i < stages.length(); i++) { + JSONObject stage = stages.getJSONObject(i); + requireAttestation( + "SUCCEEDED".equals(stage.optString("state")), + "fixture " + indexEnum + " profile stage " + i + " was not successful: " + stage); + requireAttestation( + !stage.optString("execution_type", "").trim().isEmpty(), + "fixture " + indexEnum + " profile stage " + i + " has no execution_type: " + stage); + executionTypes.put(stage.getString("execution_type")); + } + } + analyticsRouteAttestation.put("profileExecutionTypes", executionTypes); + } + + private String analyticsCanaryQuery(String indexEnum) { + String indexName = Index.valueOf(indexEnum).getName(); + switch (indexEnum) { + case "ACCOUNT": + return "source=" + indexName + " | fields account_number, firstname | head 1"; + case "FLAT_OBJECT": + return "source=" + indexName + " | fields name, status | head 1"; + default: + throw new IllegalArgumentException( + "no fixture-safe analytics canary projection is defined for " + indexEnum); + } + } + + private JSONObject runProfiledPplQuery(String query) throws IOException { + Request request = new Request("POST", QUERY_API_ENDPOINT); + request.setJsonEntity(new JSONObject().put("query", query).put("profile", true).toString()); + RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder(); + options.addHeader("Content-Type", "application/json"); + request.setOptions(options); + + Response response = client().performRequest(request); + assertEquals(200, response.getStatusLine().getStatusCode()); + return new JSONObject(getResponseBody(response, true)); + } + + private void requireEffectiveSetting(JSONObject settings, String key, String expected) { + String actual = effectiveSetting(settings, key); + requireAttestation( + expected.equals(actual), + "effective " + key + " must be " + expected + " but was \"" + actual + "\""); + } + + private void requireEffectiveSettingContains(JSONObject settings, String key, String expected) { + String actual = effectiveSetting(settings, key); + requireAttestation( + actual.contains(expected), + "effective " + key + " must contain " + expected + " but was \"" + actual + "\""); + } + + private String effectiveSetting(JSONObject settings, String key) { + String transientValue = settingInSection(settings, "transient", key); + if (!transientValue.isEmpty()) { + return transientValue; + } + String persistentValue = settingInSection(settings, "persistent", key); + if (!persistentValue.isEmpty()) { + return persistentValue; + } + return settingInSection(settings, "defaults", key); + } + + private String settingInSection(JSONObject settings, String section, String key) { + JSONObject values = settings.optJSONObject(section); + return values == null ? "" : values.optString(key, ""); + } + + private void requireIndexSetting( + String indexName, JSONObject settings, String key, String expected) { + String actual = settings.optString(key, ""); + requireAttestation( + expected.equals(actual), + "fixture " + + indexName + + " setting " + + key + + " must be " + + expected + + " but was \"" + + actual + + "\""); + } + + private void requireIndexSettingContains( + String indexName, JSONObject settings, String key, String expected) { + String actual = settings.optString(key, ""); + requireAttestation( + actual.contains(expected), + "fixture " + + indexName + + " setting " + + key + + " must contain " + + expected + + " but was \"" + + actual + + "\""); + } + + private int analyticsShardCount() { + int shardCount = Integer.parseInt(System.getProperty(ANALYTICS_SHARD_COUNT_PROPERTY, "1")); + requireAttestation(shardCount > 0, ANALYTICS_SHARD_COUNT_PROPERTY + " must be positive"); + return shardCount; + } + + private static void requireAttestation(boolean condition, String message) { + if (!condition) { + throw new IllegalStateException(message); + } + } + + @FunctionalInterface + private interface AttestationCheck { + void run() throws Exception; + } + // --- grammar bundle export ------------------------------------------------- /** @@ -653,22 +1505,26 @@ private void exportGrammarArtifacts(List failures) { writeTargetManifest("", failures); return; } + String grammarHash = ""; + String bundleName = ""; try { Response response = client().performRequest(new Request("GET", GRAMMAR_API_ENDPOINT)); String bundleBody = getResponseBody(response, true); - Files.write(Paths.get(bundlePath), bundleBody.getBytes(StandardCharsets.UTF_8)); - JSONObject bundle = new JSONObject(bundleBody); - String grammarHash = bundle.optString("grammarHash", ""); - writeTargetManifest(grammarHash, Paths.get(bundlePath).getFileName().toString(), failures); + grammarHash = bundle.optString("grammarHash", ""); + Files.write(Paths.get(bundlePath), bundleBody.getBytes(StandardCharsets.UTF_8)); + bundleName = Paths.get(bundlePath).getFileName().toString(); log("_grammar", "export", "wrote candidate bundle (" + grammarHash + ") to " + bundlePath); } catch (Exception e) { failures.add( "[grammar-export] failed to fetch/write " + GRAMMAR_API_ENDPOINT + ": " + e.getMessage()); + } finally { + // Route and attestation identity remain available even when the grammar + // endpoint or bundle write fails. + writeTargetManifest(grammarHash, bundleName, failures); } } - /** Target manifest for a leg with no grammar bundle (compiled surface / local run). */ /** * True when a failure is the cluster rejecting {@code plugins.calcite.enabled} because it does * not know that setting — i.e. a pre-Calcite (2.x) engine. @@ -697,9 +1553,8 @@ private void writeTargetManifest(String grammarHash, List failures) { } /** - * Write {@code ppl.lint.target}: the engine version, the grammar hash when there is one, and the - * bundle filename when one was exported. Every consumer keys on {@code engineVersion}, so this is - * written whether or not a bundle exists. + * Write target schema v2 with engine, grammar, execution route, storage, shard count, and (for + * analytics) route attestation identity. */ private void writeTargetManifest(String grammarHash, String bundleName, List failures) { String targetPath = System.getProperty("ppl.lint.target"); @@ -709,9 +1564,24 @@ private void writeTargetManifest(String grammarHash, String bundleName, List loadScheduledContracts() throws IOException { List result = new ArrayList<>(); + Set ruleIds = new LinkedHashSet<>(); for (String fileName : manifestContractNames()) { JSONObject contract = loadContractFile(CONTRACT_DIR + "/" + fileName); + String ruleId = contract.getString("ruleId"); + if (!ruleIds.add(ruleId)) { + throw new IOException("contract manifest contains duplicate ruleId \"" + ruleId + "\""); + } String contractSchedule = contract.optString("schedule", "pr"); if ("pr".equals(schedule) && !"pr".equals(contractSchedule)) { continue; // PR runs only PR-scheduled contracts; nightly runs all. @@ -908,8 +1783,13 @@ private List manifestContractNames() throws IOException { JSONObject manifest = loadContractFile(MANIFEST); JSONArray contracts = manifest.getJSONArray("contracts"); List names = new ArrayList<>(); + Set unique = new LinkedHashSet<>(); for (int i = 0; i < contracts.length(); i++) { - names.add(contracts.getString(i)); + String name = contracts.getString(i); + if (!unique.add(name)) { + throw new IOException("contract manifest contains duplicate file \"" + name + "\""); + } + names.add(name); } return names; } @@ -950,19 +1830,16 @@ private JSONObject reportEntry( .put("queryName", queryName) .put("role", role) .put("query", query) - .put("kind", kind); + .put("kind", kind) + .put("executionBackend", executionBackend.id); } - private void writeReport(JSONArray report) { + private void writeReport(JSONArray report) throws IOException { String target = System.getProperty("ppl.lint.report"); if (target == null || target.isEmpty()) { return; } - try { - Files.write(Paths.get(target), report.toString(2).getBytes(StandardCharsets.UTF_8)); - } catch (IOException e) { - System.err.println("[ppl-lint] could not write backend report to " + target + ": " + e); - } + Files.write(Paths.get(target), report.toString(2).getBytes(StandardCharsets.UTF_8)); } private void log(String ruleId, String caseId, String message) { @@ -970,4 +1847,27 @@ private void log(String ruleId, String caseId, String message) { String.format( Locale.ROOT, "[ppl-lint-backend-contract] %s/%s: %s", ruleId, caseId, message)); } + + private enum ExecutionBackend { + STANDARD("standard", "lucene"), + ANALYTICS("analytics", "composite-parquet"); + + private final String id; + private final String storage; + + ExecutionBackend(String id, String storage) { + this.id = id; + this.storage = storage; + } + + private static ExecutionBackend parse(String value) { + for (ExecutionBackend backend : values()) { + if (backend.id.equals(value)) { + return backend; + } + } + throw new IllegalArgumentException( + EXECUTION_BACKEND_PROPERTY + " must be standard or analytics but was \"" + value + "\""); + } + } } diff --git a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json index e1063eddc7d..bc30f5ecb77 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/division-by-zero.spec.json @@ -1,7 +1,7 @@ { "schemaVersion": 3, "ruleId": "division-by-zero", - "note": "The detector deliberately flags only `/`: division_by_zero.ts pins DIVISION_OPERATOR to \"/\" because modulo-by-zero was never verified live. `modulo-by-zero-not-flagged` is therefore a CONTROL \u2014 it documents that boundary rather than asserting a gap.", + "note": "The detector flags both division and modulo by a literal zero because both operations return null silently. The backend result-shape oracle verifies that behavior independently for each operator.", "grammarSurface": "both", "schedule": "pr", "wiring": { @@ -39,8 +39,8 @@ "role": "control", "query": "source={{index}} | eval ratio = balance / 2 | fields ratio | head 1" }, - "modulo-by-zero-not-flagged": { - "role": "control", + "modulo-by-zero-literal": { + "role": "trigger", "query": "source={{index}} | eval m = balance % 0 | fields m | head 1" } }, @@ -80,8 +80,9 @@ } } }, - "modulo-by-zero-not-flagged": { - "detectorCount": 0, + "modulo-by-zero-literal": { + "detectorCount": 1, + "severity": "warning", "backend": { "kind": "result-shape", "httpStatus": 200, diff --git a/scripts/ppl-lint-rule-validation.sh b/scripts/ppl-lint-rule-validation.sh index 295168f8aad..11e77f1e4ba 100755 --- a/scripts/ppl-lint-rule-validation.sh +++ b/scripts/ppl-lint-rule-validation.sh @@ -36,6 +36,13 @@ # # # Run the full nightly corpus (all rules + coverage assertion) # PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh +# +# # Run the same corpus through composite/Parquet + DataFusion +# RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh +# +# # Pass local analytics plugin ZIP overrides through to Gradle +# RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh \ +# -PanalyticsEngineZip=/path/to/analytics-engine.zip set -euo pipefail @@ -50,6 +57,7 @@ DETECTOR_SCRIPT="$SQL_ROOT/scripts/ppl-lint/run-frontend-contract.mjs" IT_CLASS="org.opensearch.sql.calcite.remote.PplLintRuleValidationIT" # pr (fast, blocking subset) or nightly (full corpus + coverage assertion). PPL_LINT_SCHEDULE="${PPL_LINT_SCHEDULE:-pr}" +RUN_ANALYTICS="${RUN_ANALYTICS:-0}" # Candidate artifacts the backend half exports and the detector half consumes. GRAMMAR_BUNDLE="$SQL_ROOT/ppl-grammar-bundle.json" @@ -60,12 +68,30 @@ DETECTOR_REPORT="$SQL_ROOT/detector-report.json" log() { echo "[ppl-lint-rule-validation] $*"; } run_backend() { - log "Running backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" - ./gradlew :integ-test:integTest --tests "$IT_CLASS" \ - -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" \ - -Dppl.lint.report="$BACKEND_REPORT" \ - -Dppl.lint.grammar.bundle="$GRAMMAR_BUNDLE" \ + local backend="standard" + local gradle_args=( + :integ-test:integTest + --tests "$IT_CLASS" + ) + if [[ "$RUN_ANALYTICS" == "1" ]]; then + backend="analytics" + gradle_args=(:integ-test:analyticsEnginePplLintIT) + # The checked-in schema-v3 contracts intentionally have no analytics + # oracles yet. Execute them once and retain their raw observations without + # borrowing the standard route's oracle. + gradle_args+=(-Dppl.lint.observe.only=true) + fi + + log "Running $backend backend integration test: $IT_CLASS (schedule=$PPL_LINT_SCHEDULE)" + gradle_args+=( + -Dppl.lint.schedule="$PPL_LINT_SCHEDULE" + -Dppl.lint.execution_backend="$backend" + -Dppl.lint.sql_sha="$(git rev-parse HEAD)" + -Dppl.lint.report="$BACKEND_REPORT" + -Dppl.lint.grammar.bundle="$GRAMMAR_BUNDLE" -Dppl.lint.target="$TARGET_MANIFEST" + ) + ./gradlew "${gradle_args[@]}" "$@" log "Backend integration test passed. Exported: $(basename "$GRAMMAR_BUNDLE"), $(basename "$TARGET_MANIFEST")." } @@ -93,13 +119,15 @@ run_detector() { PPL_LINT_TARGET_MANIFEST="$TARGET_MANIFEST" \ PPL_LINT_BACKEND_REPORT="$BACKEND_REPORT" \ PPL_LINT_REPORT="$DETECTOR_REPORT" \ + PPL_LINT_OBSERVE_ONLY="$RUN_ANALYTICS" \ + PPL_LINT_OBSERVE_ANALYTICS="$RUN_ANALYTICS" \ node -r ./src/setup_node_env "$DETECTOR_SCRIPT" ) log "Detector validation passed." } if [[ "${SKIP_BACKEND:-0}" != "1" ]]; then - run_backend + run_backend "$@" else log "SKIP_BACKEND=1 — skipping the SQL backend integration test (using existing artifacts)." fi diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index 688d3439cc7..db60cbe23ca 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -10,6 +10,7 @@ or a semantic change makes a flagged query valid) without touching OSD. Neither repository's own unit tests catch that. This check does. - **Design:** `ppl-lint-ci-validation-design.md` +- **Analytics rollout:** [`docs/dev/ppl-lint-analytics-engine-ci-validation.md`](../../docs/dev/ppl-lint-analytics-engine-ci-validation.md) - **Workflow:** [`.github/workflows/ppl-lint-rule-validation.yml`](../../.github/workflows/ppl-lint-rule-validation.yml) - **Contracts:** [`integ-test/src/test/resources/ppl-lint/contracts/`](../../integ-test/src/test/resources/ppl-lint/contracts) @@ -26,8 +27,8 @@ backend-validation ──(target.json, ppl-grammar-bundle.json, backend-report.j the Gradle test cluster, runs each contract's trigger/control queries against `POST /_plugins/_ppl`, and — while the cluster is alive — exports: - `ppl-grammar-bundle.json` — the candidate runtime grammar (`GET /_plugins/_ppl/_grammar`); - - `target.json` — `{ engineVersion, grammarHash, grammarBundle }`; - - `backend-report.json` — the observed HTTP behavior per query. + - `target.json` — schema-v2 engine, grammar, execution-backend, storage, and route identity; + - `backend-report.json` — the observed HTTP behavior and execution backend per query. 2. **detector-validation** (`ubuntu-latest`). Checks out and bootstraps OSD as a Node code dependency (no OSD server, no Monaco, no browser), then runs [`run-frontend-contract.mjs`](run-frontend-contract.mjs). That runner @@ -87,6 +88,14 @@ OSD_REF= ./scripts/ppl-lint-rule-validation.sh # Full nightly corpus + coverage assertion. PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh +# Run the corpus through the full composite/Parquet + DataFusion stack. +RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh + +# Use locally built analytics plugins (all trailing arguments pass to Gradle). +RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh \ + -PanalyticsEngineZip=/path/to/analytics-engine.zip \ + -PnativeLibPath=/path/to/native/release + # Re-run only one half (detector needs the backend artifacts to exist). SKIP_DETECTOR=1 ./scripts/ppl-lint-rule-validation.sh SKIP_BACKEND=1 ./scripts/ppl-lint-rule-validation.sh @@ -106,12 +115,12 @@ writes `detector-report.json`. | `PPL_LINT_CONTRACT_DIR` | directory of `*.spec.json` + `manifest.json` | | `PPL_LINT_SCHEDULE` | `pr` or `nightly` | | `PPL_LINT_GRAMMAR_BUNDLE` | candidate `ppl-grammar-bundle.json` (required; no compiled fallback) | -| `PPL_LINT_TARGET_MANIFEST` | `target.json` (engine version + grammar hash) | +| `PPL_LINT_TARGET_MANIFEST` | schema-v2 `target.json` (engine, grammar, execution backend, and storage identity) | | `PPL_LINT_BACKEND_REPORT` | `backend-report.json` (enables the differential) | | `PPL_LINT_REPORT` | where to write `detector-report.json` | | `PPL_LINT_CONTRACT_FILE` | (optional) run a single spec instead of the dir | -## Contract format (schema v3) +## Contract format (schema v3 and v4) One JSON file per rule under `contracts/`, listed in `manifest.json`. Each file has a top-level `queries` map (each `{ role: "trigger"|"control", query }`) and a @@ -120,7 +129,7 @@ backend version (zero or more than one fails before any query runs). ```jsonc { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "union-min-datasets", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -139,11 +148,17 @@ backend version (zero or more than one fails before any query runs). "queries": { "union-single-dataset": { "detectorCount": 1, "severity": "error", - "backend": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } + "backends": { + "standard": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } }, + "analytics": { "kind": "rejection", "httpStatus": 400, "body": { "status": 400, "error": { "type": "IllegalArgumentException" } } } + } }, "union-two-datasets-control": { "detectorCount": 0, - "backend": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + "backends": { + "standard": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } }, + "analytics": { "kind": "result-shape", "httpStatus": 200, "expect": { "datarowsNonEmpty": true } } + } } } } @@ -151,9 +166,18 @@ backend version (zero or more than one fails before any query runs). } ``` -`backend.kind` is one of `rejection` (contracted 4xx + error type/reason), +Schema v3's `backend` is read only as `backends.standard`; it is never an +implicit analytics oracle. Schema v4's `backends` selects the configured +`standard` or `analytics` execution backend. Missing analytics oracles are +recorded as unscored coverage during observation, including the raw backend +response needed to review a schema-v4 oracle, and are fatal before promotion to +the required check. The selected expectation must contain exactly the same query +names as the top-level `queries` map. + +Each backend `kind` is one of `rejection` (contracted 4xx + error type/reason), `result-shape` (200 with datarow expectations), or `advisory` (soft 200-only -oracle). When a behavior changes in a new version, keep **both** version-scoped +oracle). `not-applicable` requires a reason, owner, and tracking issue. When a behavior +changes in a new version, keep **both** version-scoped expectations so the nightly matrix proves the rule still fires on the old version while the candidate check proves the fix on the new one. @@ -214,7 +238,7 @@ validates every `defaultError` rule against several engine versions at once, and reports **what to change in the linter** when one disagrees. ``` -observe (matrix: 3.6.0, 3.7.0 released images + pr-build) +observe (matrix: released images + pr-build standard + pr-build analytics) └── each leg exports the same 4 artifacts as the single-version check detect (one OSD bootstrap, one detector pass per leg's grammar) └── aggregate-versions.mjs → drift-report.json + remediation report @@ -222,7 +246,10 @@ detect (one OSD bootstrap, one detector pass per leg's grammar) Released legs run the official `opensearchproject/opensearch:` image, which bundles the matching `opensearch-sql` plugin, so no old branch is built. The -`pr-build` leg is the same Gradle test cluster the single-version check uses. Both +`pr-build` leg is the same Gradle test cluster the single-version check uses. The +`pr-build-analytics` leg installs the full Arrow, analytics, composite, Parquet, +Lucene-backend, and DataFusion-backend stack and fails unless fixture settings, +explain output, and a profiled canary attest the route. These legs run the **same** contract oracle (`PplLintRuleValidationIT`) with `-Dppl.lint.observe.only=true`, which records real behavior instead of asserting against expectations — on an older engine a mismatch is the signal being @@ -263,7 +290,8 @@ cells read `n/a (surface)`, and a rule whose every case is inert is `n/a` — no is nothing to re-run). Two legs may share an engine version while validating different surfaces, so the -matrix is keyed on the **leg label**, not the version. +matrix is keyed on the **leg label**, grammar surface, and execution backend, not +the version alone. Each contract declares the surface(s) it was verified against, and a contract is only scored on a matching leg — `"both"` opts into either. Judged on a surface it @@ -304,6 +332,7 @@ Every finding names a drift class, the evidence, and one remediation action: | `version-scope-rule` | the engine relaxed (or never had) the behavior on some versions | `appliesTo.minVersion` / `maxVersion` in `rules_catalog.json` — or `enabled: false` if no supported engine rejects it any more | | `update-detector` | the detector regressed, went too broad, or its grammar anchor was renamed | the rule's detector `.ts` (named in the finding) | | `update-contract` | the linter is right and only the pinned expectation is stale | the `expectations[]` entry for that version | +| `align-execution-backends` | standard and analytics disagree for the same SQL version and grammar | reconcile the detector with both routes or add a reliable backend signal to OSD | Drift classes: `grammar-rule-missing` (a parser rule the detector walks was renamed or removed — the finding names the closest current rule names), @@ -311,7 +340,9 @@ renamed or removed — the finding names the closest current rule names), verdict flipped), `engine-message-changed` (same verdict, reworded error), `detector-silent` / `detector-noisy` (false negative / false positive), `version-scope-too-narrow` (the engine rejects but the rule is scoped away from -that version, so users see no diagnostic), and `severity-mismatch`. +that version, so users see no diagnostic), `execution-backend-divergence` (same +version, different route verdict), and `severity-mismatch`. Backend divergence +never recommends changing a version range. #### Full vs partial relaxation: scope the rule, or narrow the detector? @@ -410,6 +441,14 @@ mkdir -p legs/3.7.0 -Dppl.lint.grammar.bundle=$PWD/legs/3.7.0/ppl-grammar-bundle.json \ -Dppl.lint.target=$PWD/legs/3.7.0/target.json +# Observe the PR build through composite/Parquet + DataFusion. +mkdir -p legs/pr-build-analytics +./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.schedule=nightly -Dppl.lint.observe.only=true \ + -Dppl.lint.report=$PWD/legs/pr-build-analytics/backend-report.json \ + -Dppl.lint.grammar.bundle=$PWD/legs/pr-build-analytics/ppl-grammar-bundle.json \ + -Dppl.lint.target=$PWD/legs/pr-build-analytics/target.json + # Lint each leg's grammar from an OSD checkout (writes detector-report.json), # then compare every version at once: node scripts/ppl-lint/aggregate-versions.mjs \ diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 8100829eb9d..76f64e31348 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -114,12 +114,41 @@ function writeLeg({ version, cases, parserRuleNames = ['unionCommand', 'unionDataset'], - defaultErrorRules, + defaultErrorRules = [SPEC.ruleId], + executionBackend = 'standard', + grammarHash = `sha256:${version}`, + surface = 'runtime-bundle', + explicitIdentity = true, }) { const dir = makeTmp(`ppl-lint-leg-${version}-`); + const target = { + engineVersion: version, + grammarHash, + ...(explicitIdentity + ? { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + executionBackend, + storage: executionBackend === 'analytics' ? 'composite-parquet' : 'lucene', + shardCount: 1, + ...(executionBackend === 'analytics' + ? { + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + } + : {}), + } + : {}), + }; fs.writeFileSync( path.join(dir, 'target.json'), - JSON.stringify({ engineVersion: version, grammarHash: `sha256:${version}` }) + JSON.stringify(target) ); fs.writeFileSync( path.join(dir, 'ppl-grammar-bundle.json'), @@ -137,6 +166,9 @@ function writeLeg({ expected: role === 'trigger' ? 1 : 0, actual: c.detector, severities: c.severities || (c.detector > 0 ? ['error'] : []), + severityMatched: c.severityMatched ?? true, + messageMatched: c.messageMatched ?? true, + ...(explicitIdentity ? { executionBackend } : {}), }); backend.push({ ruleId: SPEC.ruleId, @@ -144,15 +176,31 @@ function writeLeg({ role, rejected: !!c.rejected, observed: { - httpStatus: c.rejected ? 400 : 200, + httpStatus: c.httpStatus || (c.rejected ? 400 : 200), rejected: !!c.rejected, ...(c.rejected ? { type: c.type || REJECTION.type, reason: c.reason || REJECTION.reason } : {}), }, + ...(c.outcome ? { outcome: c.outcome } : {}), + ...(c.error ? { error: c.error } : {}), + ...(explicitIdentity ? { executionBackend } : {}), }); } + const detectorIdentity = explicitIdentity + ? { + schemaVersion: 2, + executionBackend, + engineVersion: version, + grammarHash, + } + : {}; fs.writeFileSync( path.join(dir, 'detector-report.json'), - JSON.stringify({ results, ...(defaultErrorRules ? { defaultErrorRules } : {}) }) + JSON.stringify({ + ...detectorIdentity, + surface, + results, + ...(defaultErrorRules !== null ? { defaultErrorRules } : {}), + }) ); fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); return dir; @@ -163,7 +211,8 @@ function run({ contracts, legs, extraArgs = [] }) { const outDir = makeTmp('ppl-lint-out-'); const out = path.join(outDir, 'drift-report.json'); const args = [SCRIPT, '--contracts', contracts, '--out', out]; - for (const [version, dir] of Object.entries(legs)) { + const entries = Array.isArray(legs) ? legs : Object.entries(legs); + for (const [version, dir] of entries) { args.push('--leg', `${version}=${dir}`); } args.push(...extraArgs); @@ -186,6 +235,43 @@ function healthyLegs() { }; } +function writeSchema4Contracts({ includeAnalytics = true } = {}) { + const routeOracles = (standard, analytics) => ({ + standard, + ...(includeAnalytics ? { analytics } : {}), + }); + return writeContracts({ + schemaVersion: 4, + expectations: [ + { + version: '>=3.7.0', + engine: 'calcite', + queries: { + trigger: { + detectorCount: 1, + severity: 'error', + backends: routeOracles( + { + kind: 'rejection', + httpStatus: 400, + body: { status: 400, error: REJECTION }, + }, + { kind: 'result-shape', httpStatus: 200 } + ), + }, + control: { + detectorCount: 0, + backends: routeOracles( + { kind: 'result-shape', httpStatus: 200 }, + { kind: 'result-shape', httpStatus: 200 } + ), + }, + }, + }, + ], + }); +} + test('all versions agreeing exits 0 and reports no drift', () => { const { status, report, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); assert.equal(status, 0); @@ -197,6 +283,460 @@ test('all versions agreeing exits 0 and reports no drift', () => { assert.ok(report.matrix.every((m) => m.status === 'agree')); }); +test('same-version standard and analytics verdicts are classified as backend divergence', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report, stdout } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build', analytics], + ], + }); + + assert.equal(status, 1); + assert.equal(report.schemaVersion, 2); + assert.equal(report.legs.length, 2); + assert.equal(new Set(report.legs.map((leg) => leg.key)).size, 2); + assert.deepEqual( + report.legs.map((leg) => leg.executionBackend).sort(), + ['analytics', 'standard'] + ); + assert.equal(report.backendPairs.length, 1); + assert.ok(report.matrix.every((row) => row.status === 'drift')); + assert.ok(report.matrix.every((row) => row.key.includes(row.executionBackend))); + + const divergence = report.drifts.find( + (drift) => drift.driftClass === 'execution-backend-divergence' + ); + assert.ok(divergence); + assert.deepEqual(divergence.executionBackends, ['standard', 'analytics']); + assert.match(divergence.key, /standard-vs-analytics/); + assert.equal(divergence.remediation.action, 'align-execution-backends'); + assert.doesNotMatch(divergence.remediation.detail, /maxVersion|minVersion|scope/i); + assert.equal( + report.drifts.filter( + (drift) => + drift.driftClass === 'engine-relaxed' || drift.driftClass === 'engine-tightened' + ).length, + 0, + 'route differences must not be rendered as product-version drift' + ); + assert.match(stdout, /`3\.8\.0`
      standard/); + assert.match(stdout, /`3\.8\.0`
      analytics/); +}); + +test('schema-v3 analytics has explicit backend-oracle coverage holes', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report, stdout } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 2); + assert.ok(report.coverageHoles.every((hole) => hole.executionBackend === 'analytics')); + assert.ok(report.coverageHoles.every((hole) => hole.kind === 'backend-oracle')); + assert.ok(report.coverageHoles.every((hole) => /standard-only/.test(hole.reason))); + assert.equal(report.matrix[0].status, 'uncovered'); + assert.equal(report.drifts.length, 0); + assert.match(stdout, /schema-v3 backend oracles are standard-only/); +}); + +test('analytics observation mode reports schema-v3 coverage without failing', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 0); + assert.equal(report.result.enforcedCoverageHoles, 0); + assert.equal(report.result.observedAnalyticsFindings, 2); + assert.ok(report.coverageHoles.every((hole) => hole.blocking === false)); +}); + +test('schema-v3 raw analytics observations still expose backend divergence', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ...entry, + kind: 'coverage-missing', + outcome: 'coverage-missing', + coverage: 'missing', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 0); + assert.equal( + report.drifts.filter((drift) => drift.driftClass === 'execution-backend-divergence') + .length, + 1 + ); + assert.equal(report.result.observedAnalyticsFindings, 3); +}); + +test('backend divergence cannot hide a standard-route regression', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + + const { status, report } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 1); + assert.ok( + report.drifts.some( + (drift) => + drift.executionBackend === 'standard' && + drift.driftClass === 'engine-relaxed' + ), + 'the blocking standard-route regression must survive paired-route classification' + ); + assert.ok( + report.drifts.some( + (drift) => drift.driftClass === 'execution-backend-divergence' + ) + ); + assert.ok(report.result.enforcedDriftCount > 0); +}); + +test('not-applicable cannot waive an enforced analytics backend oracle', () => { + const contracts = writeSchema4Contracts(); + const contractFile = path.join(contracts, 'union.spec.json'); + const contract = JSON.parse(fs.readFileSync(contractFile, 'utf8')); + for (const query of Object.values(contract.expectations[0].queries)) { + query.backends.analytics = { + kind: 'not-applicable', + reason: 'analytics fixture is not supported yet', + owner: '@analytics-team', + issue: 'https://example.test/issues/42', + }; + } + fs.writeFileSync(contractFile, JSON.stringify(contract)); + + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: entry.executionBackend, + kind: 'not-applicable', + outcome: 'not-applicable', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts, + legs: [['pr-build-analytics', analytics]], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedCoverageHoles, 2); + assert.ok(report.coverageHoles.every((hole) => hole.issue.endsWith('/42'))); + assert.equal(report.matrix[0].status, 'uncovered'); +}); + +test('analytics coverage gaps cannot hide missing raw observations', () => { + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const backendFile = path.join(analytics, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).map((entry) => ({ + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: 'analytics', + kind: 'coverage-missing', + outcome: 'error', + error: 'connect timeout', + })); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: [['pr-build-analytics', analytics]], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(report.inconclusive[0].reasons.join(' '), /no engine verdict/); +}); + +test('paired detector reports must be identical across execution backends', () => { + const grammarHash = 'sha256:shared-runtime-grammar'; + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash, + cases: { + trigger: { detector: 1, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const detectorFile = path.join(analytics, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results.find((entry) => entry.queryName === 'trigger').actual = 0; + detector.results.find((entry) => entry.queryName === 'trigger').severities = []; + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build-analytics', analytics], + ], + extraArgs: ['--observe-analytics'], + }); + + assert.equal(status, 2); + assert.match(stderr, /detector parity failed for union-min-datasets::trigger/); +}); + +test('target and detector execution identities must match', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.executionBackend = 'standard'; + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [['pr-build-analytics', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /detector report executionBackend "standard" does not match target "analytics"/); +}); + +test('unknown target execution backends are rejected', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + cases: { trigger: { detector: 1, rejected: true } }, + }); + const targetFile = path.join(dir, 'target.json'); + const target = JSON.parse(fs.readFileSync(targetFile, 'utf8')); + target.executionBackend = 'experimental'; + fs.writeFileSync(targetFile, JSON.stringify(target)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['pr-build', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /must be "standard" or "analytics"/); +}); + +test('every backend row must match its target execution identity', () => { + const dir = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend[0].executionBackend = 'standard'; + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [['pr-build-analytics', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /backend report row .* does not match target "analytics"/); +}); + +test('duplicate backend row keys are rejected instead of overwritten', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')); + backend.push({ ...backend[0] }); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['3.8.0', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate backend report key "union-min-datasets::trigger"/); +}); + +test('duplicate detector row keys are rejected instead of selecting the first', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results.push({ ...detector.results[0] }); + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [['3.8.0', dir]], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate detector report key "union-min-datasets::trigger"/); +}); + +test('duplicate backend-qualified leg identities are rejected', () => { + const dir = writeLeg({ + version: '3.8.0', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: [ + ['3.8.0', dir], + ['3.8.0', dir], + ], + }); + assert.equal(status, 2); + assert.match(stderr, /duplicate leg identity/); +}); + +test('paired standard and analytics legs require the same runtime grammar hash', () => { + const standard = writeLeg({ + version: '3.8.0', + executionBackend: 'standard', + explicitIdentity: true, + grammarHash: 'sha256:standard', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const analytics = writeLeg({ + version: '3.8.0', + executionBackend: 'analytics', + grammarHash: 'sha256:analytics', + cases: { trigger: { detector: 1, rejected: true } }, + }); + const { status, stderr } = run({ + contracts: writeSchema4Contracts(), + legs: [ + ['pr-build', standard], + ['pr-build', analytics], + ], + }); + assert.equal(status, 2); + assert.match(stderr, /different grammar hashes/); +}); + test('a version where only one engine relaxed is red, and names just that version', () => { const legs = healthyLegs(); // 3.8 now accepts what 3.7 still rejects, while the detector keeps flagging. @@ -215,6 +755,54 @@ test('a version where only one engine relaxed is red, and names just that versio assert.equal(report.matrix.find((m) => m.version === '3.7.0').status, 'agree'); }); +test('a changed rejection HTTP status is semantic drift, not agreement', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true, httpStatus: 500 }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['3.8.0', leg]], + }); + + assert.equal(status, 1); + const drift = report.drifts.find( + (entry) => entry.driftClass === 'backend-oracle-mismatch' + ); + assert.ok(drift); + assert.match(drift.evidence, /HTTP status changed from 400 to 500/); + assert.equal(drift.remediation.action, 'review-backend-oracle'); +}); + +test('a same-verdict result-shape mismatch cannot pass aggregation', () => { + const leg = writeLeg({ + version: '3.8.0', + cases: { + trigger: { detector: 1, rejected: true }, + control: { + detector: 0, + rejected: false, + outcome: 'observed-mismatch', + error: 'expected non-empty datarows', + }, + }, + }); + const { status, report } = run({ + contracts: writeContracts(), + legs: [['3.8.0', leg]], + }); + + assert.equal(status, 1); + const drift = report.drifts.find( + (entry) => entry.driftClass === 'backend-oracle-mismatch' + ); + assert.ok(drift); + assert.match(drift.evidence, /expected non-empty datarows/); +}); + // --- partial vs full relaxation, end to end --------------------------------- // // Driven through the real script because the bug this guards is in the AGGREGATION: @@ -460,12 +1048,55 @@ test('a census matching the manifest keeps the check green', () => { assert.equal(report.result.missingContractCount, 0); }); -test('a legacy detector report without a census warns instead of failing', () => { - // Older detector builds do not emit defaultErrorRules; the aggregator must say - // so out loud rather than quietly reporting full coverage. - const { status, stdout } = run({ contracts: writeContracts(), legs: healthyLegs() }); - assert.equal(status, 0); - assert.match(stdout, /no detector leg reported a defaultErrorRules census/); +test('a schema-v2 detector report without a census fails closed', () => { + const dir = writeLeg({ + version: '3.8.0', + defaultErrorRules: null, + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /defaultErrorRules must be a JSON array/); +}); + +test('a schema-v2 detector report with an unknown grammar surface fails closed', () => { + const dir = writeLeg({ + version: '3.8.0', + surface: 'unknown-surface', + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /surface must be "runtime-bundle" or "compiled-simplified"/); +}); + +test('a schema-v2 detector census rejects duplicate rule identities', () => { + const dir = writeLeg({ + version: '3.8.0', + defaultErrorRules: [SPEC.ruleId, SPEC.ruleId], + cases: { + trigger: { detector: 1, rejected: true }, + control: { detector: 0, rejected: false }, + }, + }); + const { status, stderr } = run({ + contracts: writeContracts(), + legs: { '3.8.0': dir }, + }); + assert.equal(status, 2); + assert.match(stderr, /defaultErrorRules contains duplicate rule/); }); // --- "we don't know" must never render as "it's fine" ------------------------- @@ -478,7 +1109,14 @@ function writeLegWithTransportError({ version, erroredQuery, cases }) { entry.queryName === erroredQuery ? // Exactly what the IT writes on a transport failure: an `error` outcome and // NO `rejected` field, because no verdict was ever received. - { ruleId: entry.ruleId, queryName: entry.queryName, role: entry.role, outcome: 'error', error: 'connect timeout' } + { + ruleId: entry.ruleId, + queryName: entry.queryName, + role: entry.role, + executionBackend: entry.executionBackend, + outcome: 'error', + error: 'connect timeout', + } : { ...entry, outcome: 'observed' } ); fs.writeFileSync(file, JSON.stringify(backend)); @@ -523,11 +1161,19 @@ test('losing every trigger is inconclusive even when a control still compares', fs.writeFileSync( path.join(dir, 'backend-report.json'), JSON.stringify([ - { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', + executionBackend: 'standard', rejected: false, outcome: 'observed', observed: { httpStatus: 200, rejected: false }, @@ -549,8 +1195,22 @@ test('a leg where nothing could be compared is inconclusive, not agreement', () fs.writeFileSync( path.join(dir, 'backend-report.json'), JSON.stringify([ - { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, - { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, + { + ruleId: SPEC.ruleId, + queryName: 'control', + role: 'control', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, ]) ); const { status, report, stdout } = run({ contracts: writeContracts(), legs: { '3.7.0': dir } }); @@ -622,18 +1282,29 @@ test('an errored trigger on an out-of-scope rule does not silently pass', () => fs.writeFileSync( path.join(dir, 'backend-report.json'), JSON.stringify([ - { ruleId: SPEC.ruleId, queryName: 'trigger', role: 'trigger', outcome: 'error', error: 'timeout' }, + { + ruleId: SPEC.ruleId, + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'timeout', + }, { ruleId: SPEC.ruleId, queryName: 'control', role: 'control', + executionBackend: 'standard', rejected: false, outcome: 'observed', observed: { httpStatus: 200, rejected: false }, }, ]) ); - const { report } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + const { status, report } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); // The point is that an unobserved trigger yields no CLAIM either way: it must // not be reported as a confident out-of-scope agreement... assert.equal( @@ -643,6 +1314,37 @@ test('an errored trigger on an out-of-scope rule does not silently pass', () => ); // ...nor may it invent linter advice from a verdict that never arrived. assert.equal(report.drifts.length, 0); + assert.equal(status, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.equal(report.result.enforcedInconclusive, 1); +}); + +test('a missing detector and backend row is inconclusive even when the rule is out of scope', () => { + const dir = writeLeg({ + version: '3.6.0', + cases: { + trigger: { detector: 0, rejected: false }, + control: { detector: 0, rejected: false }, + }, + }); + const detectorFile = path.join(dir, 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(detectorFile, 'utf8')); + detector.results = detector.results.filter((entry) => entry.queryName !== 'trigger'); + fs.writeFileSync(detectorFile, JSON.stringify(detector)); + const backendFile = path.join(dir, 'backend-report.json'); + const backend = JSON.parse(fs.readFileSync(backendFile, 'utf8')).filter( + (entry) => entry.queryName !== 'trigger' + ); + fs.writeFileSync(backendFile, JSON.stringify(backend)); + + const { status, report } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); + + assert.equal(status, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); + assert.match(report.inconclusive[0].reasons.join(' '), /trigger \(no detector result\)/); }); test('an errored control cannot fail open into "widen appliesTo" advice', () => { @@ -657,11 +1359,21 @@ test('an errored control cannot fail open into "widen appliesTo" advice', () => const backend = JSON.parse(fs.readFileSync(path.join(dir, 'backend-report.json'), 'utf8')).map( (e) => e.role === 'control' - ? { ruleId: e.ruleId, queryName: e.queryName, role: e.role, outcome: 'error', error: 'timeout' } + ? { + ruleId: e.ruleId, + queryName: e.queryName, + role: e.role, + executionBackend: e.executionBackend, + outcome: 'error', + error: 'timeout', + } : { ...e, outcome: 'observed' } ); fs.writeFileSync(path.join(dir, 'backend-report.json'), JSON.stringify(backend)); - const { report, stdout } = run({ contracts: writeContracts(), legs: { '3.6.0': dir } }); + const { status, report, stdout } = run({ + contracts: writeContracts(), + legs: { '3.6.0': dir }, + }); const scoped = report.drifts.filter((d) => d.driftClass === 'version-scope-too-narrow'); assert.equal( scoped.length, @@ -669,6 +1381,8 @@ test('an errored control cannot fail open into "widen appliesTo" advice', () => 'with the control unobserved there is no evidence the command is supported, so no widening advice' ); assert.ok(!/Widen "/.test(stdout)); + assert.equal(status, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); }); test('a bad --leg argument is rejected', () => { diff --git a/scripts/ppl-lint/__tests__/annotate.test.mjs b/scripts/ppl-lint/__tests__/annotate.test.mjs index dc55503a6ad..357b046d47c 100644 --- a/scripts/ppl-lint/__tests__/annotate.test.mjs +++ b/scripts/ppl-lint/__tests__/annotate.test.mjs @@ -129,6 +129,29 @@ test('a non-enforced drift is a warning so it cannot be read as blocking', () => assert.equal(annotations[0].level, 'warning'); }); +test('backend divergence annotations name both execution routes', () => { + const annotations = buildAnnotations( + { + drifts: [ + { + ruleId: 'invalid-capture-group-name', + version: '3.8.0', + executionBackend: 'analytics', + executionBackends: ['standard', 'analytics'], + driftClass: 'execution-backend-divergence', + enforced: true, + contractFile: 'invalid-capture-group-name.spec.json', + evidence: 'standard rejected while analytics accepted', + remediation: { action: 'align-execution-backends', detail: 'Align route behavior.' }, + }, + ], + }, + { contractsDir: '/w/contracts', workspace: '/w', readFile: readStub(CONTRACT) } + ); + assert.match(annotations[0].title, /standard vs analytics/); + assert.match(annotations[0].title, /execution-backend-divergence/); +}); + test('an unvalidated rule has no file to point at', () => { const annotations = buildAnnotations( { missingContracts: [{ ruleId: 'sort-on-eval-field', reason: 'has no contract file' }] }, diff --git a/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs new file mode 100644 index 00000000000..1c8e1f4b21d --- /dev/null +++ b/scripts/ppl-lint/__tests__/assemble-run-manifest.test.mjs @@ -0,0 +1,174 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'assemble-run-manifest.mjs'); +const tmpDirs = []; + +function makeRun() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-manifest-')); + tmpDirs.push(dir); + fs.mkdirSync(path.join(dir, 'artifacts')); + return dir; +} + +function writeJson(dir, name, value) { + fs.writeFileSync(path.join(dir, 'artifacts', name), JSON.stringify(value)); +} + +function validArtifacts(dir) { + writeJson(dir, 'target.json', { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }); + writeJson(dir, 'backend-report.json', [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + rejected: false, + observed: { httpStatus: 200, rejected: false, response: { datarows: [] } }, + outcome: 'pass', + }, + ]); + writeJson(dir, 'detector-report.json', { + schemaVersion: 2, + executionBackend: 'standard', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:test', + surface: 'runtime-bundle', + defaultErrorRules: ['advisory-rule'], + results: [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + expected: 1, + actual: 1, + severities: ['warning'], + severityMatched: true, + messageMatched: true, + executionBackend: 'standard', + }, + ], + }); +} + +function run(dir, extraEnv = {}) { + return spawnSync(process.execPath, [SCRIPT], { + cwd: dir, + encoding: 'utf8', + env: { + ...process.env, + BACKEND_RESULT: 'success', + DETECTOR_RESULT: 'success', + SQL_SHA: 'candidate-sql-sha', + ...extraEnv, + }, + }); +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('valid standard artifacts produce a passing schema-v2 manifest', () => { + const dir = makeRun(); + validArtifacts(dir); + const result = run(dir); + assert.equal(result.status, 0, result.stderr); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.equal(manifest.schemaVersion, 2); + assert.equal(manifest.executionBackend, 'standard'); + assert.equal(manifest.result.passed, true); + assert.deepEqual(manifest.result.artifactErrors, []); +}); + +test('a missing report fails closed while still writing the manifest', () => { + const dir = makeRun(); + validArtifacts(dir); + fs.rmSync(path.join(dir, 'artifacts', 'backend-report.json')); + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /required artifact is missing/); + const manifest = JSON.parse(fs.readFileSync(path.join(dir, 'run-manifest.json'), 'utf8')); + assert.equal(manifest.result.passed, false); +}); + +test('detector rows must have unique identities matching the target', () => { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results.push({ ...detector.results[0] }); + fs.writeFileSync(file, JSON.stringify(detector)); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /duplicate row advisory-rule::trigger/); +}); + +test('a backend row without a real verdict cannot render as acceptance', () => { + const dir = makeRun(); + validArtifacts(dir); + writeJson(dir, 'backend-report.json', [ + { + ruleId: 'advisory-rule', + queryName: 'trigger', + role: 'trigger', + executionBackend: 'standard', + outcome: 'error', + error: 'connect timeout', + }, + ]); + + const result = run(dir); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /did not pass its oracle/); +}); + +test('an accepted advisory trigger is summarized from its backend outcome, not its role', () => { + const dir = makeRun(); + validArtifacts(dir); + const summary = path.join(dir, 'summary.md'); + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.equal(result.status, 0, result.stderr); + const markdown = fs.readFileSync(summary, 'utf8'); + assert.match(markdown, /advisory-rule.*accepted.*Pass/); +}); + +test('detector severity and message mismatches fail the manifest and summary', () => { + for (const field of ['severityMatched', 'messageMatched']) { + const dir = makeRun(); + validArtifacts(dir); + const file = path.join(dir, 'artifacts', 'detector-report.json'); + const detector = JSON.parse(fs.readFileSync(file, 'utf8')); + detector.results[0][field] = false; + fs.writeFileSync(file, JSON.stringify(detector)); + const summary = path.join(dir, 'summary.md'); + + const result = run(dir, { GITHUB_STEP_SUMMARY: summary }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, new RegExp(`did not match its ${field === 'severityMatched' ? 'severity' : 'message'}`)); + assert.match(fs.readFileSync(summary, 'utf8'), /advisory-rule.*accepted.*Fail/); + } +}); diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs new file mode 100644 index 00000000000..6dce090d45d --- /dev/null +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -0,0 +1,469 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; + +import { + assertContractSchema, + assertExactQueryCoverage, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from '../contract-schema.mjs'; + +const QUERY = { + detectorCount: 1, + severity: 'error', + matchMessage: 'bad query', + backend: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, +}; + +function spec(schemaVersion, queryExpectation = QUERY) { + return { + schemaVersion, + ruleId: 'example-rule', + queries: { + trigger: { role: 'trigger', query: 'source={{index}} | bad' }, + control: { role: 'control', query: 'source={{index}} | head 1' }, + }, + expectations: [ + { + version: '>=3.7.0', + queries: { + trigger: queryExpectation, + control: { + detectorCount: 0, + ...(schemaVersion === 3 + ? { backend: { kind: 'result-shape', httpStatus: 200 } } + : { + backends: { + standard: { kind: 'result-shape', httpStatus: 200 }, + analytics: { kind: 'result-shape', httpStatus: 200 }, + }, + }), + }, + }, + }, + ], + }; +} + +test('target schema v2 requires and preserves explicit standard or analytics identity', () => { + for (const executionBackend of ['standard', 'analytics']) { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:abc', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend, + storage: executionBackend === 'analytics' ? 'composite-parquet' : 'lucene', + shardCount: 1, + ...(executionBackend === 'analytics' + ? { + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + } + : {}), + }); + assert.equal(target.executionBackend, executionBackend); + assert.equal(target.legacy, false); + } +}); + +test('unversioned targets cannot infer a standard execution identity', () => { + assert.throws( + () => + normalizeTarget({ + engineVersion: '3.7.0', + grammarHash: 'sha256:legacy', + }), + /target\.schemaVersion is required/ + ); +}); + +test('unknown target schema and execution backend are rejected', () => { + assert.throws( + () => + normalizeTarget({ + schemaVersion: 3, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'standard', + storage: 'lucene', + }), + /Unsupported target schemaVersion 3/ + ); + assert.throws( + () => + normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'experimental', + }), + /must be "standard" or "analytics"/ + ); +}); + +test('schema v3 resolves a standard oracle and never falls back for analytics', () => { + const contract = spec(3); + const standard = resolveBackendOracle(contract, QUERY, 'standard'); + const analytics = resolveBackendOracle(contract, QUERY, 'analytics'); + + assert.equal(standard.status, 'applicable'); + assert.equal(standard.oracle, QUERY.backend); + assert.deepEqual(standard.detector, analytics.detector); + assert.equal(analytics.status, 'coverage-missing'); + assert.equal(analytics.oracle, undefined); + assert.match(analytics.reason, /standard-only/); +}); + +test('analytics targets fail closed on storage and route attestation', () => { + const base = { + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }; + assert.equal(normalizeTarget(base).executionBackend, 'analytics'); + assert.throws( + () => normalizeTarget({ ...base, storage: 'lucene' }), + /storage must be "composite-parquet"/ + ); + const withoutStack = { ...base }; + delete withoutStack.analyticsStack; + assert.throws( + () => normalizeTarget(withoutStack), + /analyticsStack must be a JSON object/ + ); + assert.throws( + () => + normalizeTarget({ + ...base, + routeAttestation: { ...base.routeAttestation, explainVerified: false }, + }), + /explainVerified must be true/ + ); +}); + +test('schema-v2 standard targets require explicit storage and shard identity', () => { + const base = { + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }; + assert.equal(normalizeTarget(base).executionBackend, 'standard'); + assert.throws( + () => normalizeTarget({ ...base, storage: undefined }), + /storage must be "lucene"/ + ); + assert.throws( + () => normalizeTarget({ ...base, shardCount: undefined }), + /shardCount must be a positive integer/ + ); +}); + +test('schema v4 selects only the requested backend oracle', () => { + const queryExpectation = { + detectorCount: 1, + severity: 'warning', + backends: { + standard: { kind: 'rejection', httpStatus: 400, body: { status: 400 } }, + analytics: { kind: 'advisory', httpStatus: 200 }, + }, + }; + const contract = spec(4, queryExpectation); + + const standard = resolveBackendOracle(contract, queryExpectation, 'standard'); + const analytics = resolveBackendOracle(contract, queryExpectation, 'analytics'); + assert.equal(standard.oracle, queryExpectation.backends.standard); + assert.equal(analytics.oracle, queryExpectation.backends.analytics); + assert.deepEqual(standard.detector, analytics.detector); +}); + +test('schema v4 reports missing route coverage without using another backend oracle', () => { + const queryExpectation = { + detectorCount: 1, + severity: 'error', + backends: { + standard: { kind: 'rejection', httpStatus: 400 }, + }, + }; + const analytics = resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'); + + assert.equal(analytics.status, 'coverage-missing'); + assert.equal(analytics.oracle, undefined); + assert.match(analytics.reason, /no analytics backend oracle/); +}); + +test('not-applicable is explicit while an absent oracle is coverage-missing', () => { + const notApplicable = { + detectorCount: 1, + backends: { + analytics: { + kind: 'not-applicable', + reason: 'fixture is unsupported', + owner: '@analytics-team', + issue: 'https://example.test/issues/1', + }, + }, + }; + const missing = { + detectorCount: 1, + backends: {}, + }; + + assert.equal( + resolveBackendOracle(spec(4, notApplicable), notApplicable, 'analytics').status, + 'not-applicable' + ); + assert.equal( + resolveBackendOracle(spec(4, missing), missing, 'analytics').status, + 'coverage-missing' + ); + assert.throws( + () => + resolveBackendOracle( + spec(4, { + detectorCount: 1, + backends: { analytics: { kind: 'not-applicable' } }, + }), + { detectorCount: 1, backends: { analytics: { kind: 'not-applicable' } } }, + 'analytics' + ), + /backend oracle\.reason/ + ); + assert.throws( + () => { + const oracle = { + detectorCount: 1, + backends: { + analytics: { + kind: 'not-applicable', + reason: 'fixture is unsupported', + issue: 'https://example.test/issues/1', + }, + }, + }; + return resolveBackendOracle(spec(4, oracle), oracle, 'analytics'); + }, + /backend oracle\.owner/ + ); +}); + +test('unknown contract schema and backend oracle kind are rejected', () => { + assert.throws(() => assertContractSchema(spec(5)), /expected 3 or 4/); + const queryExpectation = { + detectorCount: 1, + backends: { analytics: { kind: 'maybe' } }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'), + /unknown analytics backend oracle.kind/ + ); + const unknownBackend = { + detectorCount: 1, + backends: { experimental: { kind: 'advisory' } }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, unknownBackend), unknownBackend, 'analytics'), + /backends key must be "standard" or "analytics"/ + ); +}); + +test('backend oracle payloads fail closed when required shapes are malformed', () => { + const cases = [ + { + oracle: { kind: 'rejection', body: { status: 400 } }, + expected: /httpStatus/, + }, + { + oracle: { kind: 'rejection', httpStatus: 400 }, + expected: /\.body must be a JSON object/, + }, + { + oracle: { + kind: 'rejection', + httpStatus: 400, + body: { status: '400' }, + }, + expected: /\.body\.status must be an integer/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 200, + expect: { datarowsNonEmpty: 'yes' }, + }, + expected: /datarowsNonEmpty must be a boolean/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 200, + expect: { datarowsCount: -1 }, + }, + expected: /datarowsCount must be a non-negative integer/, + }, + ]; + + for (const { oracle, expected } of cases) { + const queryExpectation = { + detectorCount: 1, + backends: { analytics: oracle }, + }; + assert.throws( + () => resolveBackendOracle(spec(4, queryExpectation), queryExpectation, 'analytics'), + expected + ); + } +}); + +test('selected expectation query keys must exactly equal top-level query keys', () => { + const contract = spec(3); + assert.deepEqual( + assertExactQueryCoverage(contract, contract.expectations[0]), + ['control', 'trigger'] + ); + + const missing = structuredClone(contract.expectations[0]); + delete missing.queries.control; + assert.throws( + () => assertExactQueryCoverage(contract, missing), + /missing from expectation: control/ + ); + + const extra = structuredClone(contract.expectations[0]); + extra.queries.unknown = QUERY; + assert.throws( + () => assertExactQueryCoverage(contract, extra), + /not present in contract\.queries: unknown/ + ); + + assert.throws( + () => + assertExactQueryCoverage( + { ...contract, queries: {} }, + { ...contract.expectations[0], queries: {} } + ), + /contract\.queries must not be empty/ + ); +}); + +test('non-verdict backend states are never coerced to acceptance', () => { + assert.deepEqual(classifyBackendReportRow({ rejected: false }), { + status: 'observed', + rejected: false, + }); + for (const outcome of ['not-applicable', 'coverage-missing', 'error']) { + assert.equal( + classifyBackendReportRow({ outcome, rejected: false }).status, + outcome + ); + assert.equal( + classifyBackendReportRow({ outcome, rejected: false }).rejected, + undefined + ); + } + assert.equal(classifyBackendReportRow({ outcome: 'pass' }).status, 'error'); +}); + +test('backend report indexing rejects duplicate keys', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }); + const row = { + ruleId: 'example-rule', + queryName: 'trigger', + executionBackend: 'analytics', + rejected: true, + }; + assert.throws(() => indexBackendReport([row, { ...row }], target), /duplicate backend report key/); + assert.throws(() => indexBackendReport({}, target), /must be a JSON array/); +}); + +test('every schema-v2 backend report row must carry identity matching the target', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.8.0', + grammarHash: 'sha256:x', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + }); + const base = { ruleId: 'example-rule', queryName: 'trigger', rejected: true }; + + assert.throws(() => indexBackendReport([base], target), /missing executionBackend/); + assert.throws( + () => indexBackendReport([{ ...base, executionBackend: 'standard' }], target), + /does not match target "analytics"/ + ); + assert.equal( + indexBackendReport([{ ...base, executionBackend: 'analytics' }], target).get( + 'example-rule::trigger' + ).rejected, + true + ); +}); + +test('schema-v2 standard backend rows cannot omit identity', () => { + const target = normalizeTarget({ + schemaVersion: 2, + engineVersion: '3.7.0', + grammarHash: 'sha256:legacy', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + }); + const row = { ruleId: 'example-rule', queryName: 'trigger', rejected: true }; + + assert.throws(() => indexBackendReport([row], target), /missing executionBackend/); + assert.throws( + () => indexBackendReport([{ ...row, executionBackend: 'analytics' }], target), + /does not match target "standard"/ + ); +}); diff --git a/scripts/ppl-lint/__tests__/drift.test.mjs b/scripts/ppl-lint/__tests__/drift.test.mjs index 251ecb3c9bf..e8e3eecfde6 100644 --- a/scripts/ppl-lint/__tests__/drift.test.mjs +++ b/scripts/ppl-lint/__tests__/drift.test.mjs @@ -23,6 +23,7 @@ import { DRIFT_CLASSES, REMEDIATIONS, classifyDrift, + classifyExecutionBackendDivergence, classifyRelaxationScope, formatDriftReport, parseVersion, @@ -136,6 +137,39 @@ test('grammar-rule check is skipped when the contract declares no required rules // --- engine behavior flips ---------------------------------------------------- +test('same-candidate route differences use backend remediation, never version scoping', () => { + const drift = classifyExecutionBackendDivergence({ + ruleId: 'union-min-datasets', + version: '3.8.0', + queryName: 'union-single-dataset', + role: 'trigger', + query: 'union [ source=t ]', + standardObserved: { backendRejected: true, backendType: 'IllegalArgumentException' }, + analyticsObserved: { backendRejected: false }, + standardLeg: 'pr-build', + analyticsLeg: 'pr-build-analytics', + grammarHash: 'sha256:same', + }); + assert.equal(drift.driftClass, DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE); + assert.equal(drift.remediation.action, REMEDIATIONS.ALIGN_EXECUTION_BACKENDS); + assert.deepEqual(drift.executionBackends, ['standard', 'analytics']); + assert.doesNotMatch(drift.remediation.detail, /maxVersion|minVersion|scope/i); + assert.match(drift.evidence, /standard REJECTED.*analytics ACCEPTED/); +}); + +test('analytics oracle flips are not labeled as product-version relaxation', () => { + const drift = classifyDrift( + agreeingTrigger({ + executionBackend: 'analytics', + observed: { detectorCount: 1, severities: ['error'], backendRejected: false }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.REVIEW_BACKEND_ORACLE); + assert.notEqual(drift.driftClass, DRIFT_CLASSES.ENGINE_RELAXED); + assert.doesNotMatch(drift.remediation.detail, /maxVersion|minVersion|scope/i); +}); + test('engine relaxation with a still-firing detector demands version scoping', () => { const drift = classifyDrift( agreeingTrigger({ @@ -357,6 +391,46 @@ test('a noisy detector the engine agrees with points at the expectation', () => assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_CONTRACT); }); +test('a nonzero detector count mismatch is not reduced to flagged versus silent', () => { + const drift = classifyDrift( + agreeingTrigger({ + expected: { + detectorCount: 2, + severity: 'error', + backendKind: 'rejection', + }, + observed: { + ...agreeingTrigger().observed, + detectorCount: 1, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_COUNT_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /expected exactly 2.*emitted 1/); +}); + +test('a detector message mismatch is classified independently of count and severity', () => { + const drift = classifyDrift( + agreeingTrigger({ + expected: { + detectorCount: 1, + severity: 'error', + matchMessage: 'requires at least two datasets', + backendKind: 'rejection', + }, + observed: { + ...agreeingTrigger().observed, + severityMatched: true, + messageMatched: false, + }, + }) + ); + assert.equal(drift.driftClass, DRIFT_CLASSES.DETECTOR_MESSAGE_MISMATCH); + assert.equal(drift.remediation.action, REMEDIATIONS.UPDATE_DETECTOR); + assert.match(drift.evidence, /requires at least two datasets/); +}); + // --- severity ---------------------------------------------------------------- test('a downgraded severity is caught even when the count is right', () => { diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index d7d6b25731f..e5dd0510ca6 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -34,8 +34,18 @@ import fs from 'fs'; import path from 'path'; import { emitAnnotations } from './annotate.mjs'; +import { + assertContractSchema, + assertExactQueryCoverage, + assertExecutionBackend, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; import { classifyDrift, + classifyExecutionBackendDivergence, classifyGrammarDrift, classifyRelaxationScope, DRIFT_CLASSES, @@ -55,7 +65,14 @@ function fatal(message) { } function parseArgs(argv) { - const args = { legs: [], contracts: '', out: 'drift-report.json', summary: '', allRules: false }; + const args = { + legs: [], + contracts: '', + out: 'drift-report.json', + summary: '', + allRules: false, + observeAnalytics: false, + }; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; const next = () => { @@ -76,6 +93,8 @@ function parseArgs(argv) { args.summary = next(); } else if (arg === '--all-rules') { args.allRules = true; + } else if (arg === '--observe-analytics') { + args.observeAnalytics = true; } else { fatal(`unknown argument "${arg}"`); } @@ -99,12 +118,206 @@ function readJson(file, { optional = false } = {}) { return undefined; } +function artifactFatal(file, error) { + fatal(`invalid ${file}: ${error.message}`); +} + +function reportRowKey(entry, label) { + if (!entry || typeof entry !== 'object' || Array.isArray(entry)) { + throw new TypeError(`${label} row must be a JSON object`); + } + if (typeof entry.ruleId !== 'string' || entry.ruleId.length === 0) { + throw new TypeError(`${label} row.ruleId must be a non-empty string`); + } + if (typeof entry.queryName !== 'string' || entry.queryName.length === 0) { + throw new TypeError(`${label} row.queryName must be a non-empty string`); + } + return `${entry.ruleId}::${entry.queryName}`; +} + +function rowExecutionBackend(entry, target, label, key) { + const hasIdentity = Object.prototype.hasOwnProperty.call(entry, 'executionBackend'); + if (!hasIdentity && !target.legacy) { + throw new Error(`${label} row ${key} is missing executionBackend for a schema-v2 target`); + } + const executionBackend = hasIdentity + ? assertExecutionBackend(entry.executionBackend, `${label} row ${key}.executionBackend`) + : 'standard'; + if (executionBackend !== target.executionBackend) { + throw new Error( + `${label} row ${key} executionBackend "${executionBackend}" does not match target ` + + `"${target.executionBackend}"` + ); + } + return executionBackend; +} + +function validateOptionalRowIdentity(entry, target, label, key) { + for (const field of ['engineVersion', 'grammarHash']) { + if ( + Object.prototype.hasOwnProperty.call(entry, field) && + entry[field] !== target[field] + ) { + throw new Error( + `${label} row ${key} ${field} ${JSON.stringify(entry[field])} does not match target ` + + `${JSON.stringify(target[field])}` + ); + } + } +} + +function normalizeDetectorReport(detector, target) { + if (!detector || typeof detector !== 'object' || Array.isArray(detector)) { + throw new TypeError('detector report must be a JSON object'); + } + + const hasIdentity = Object.prototype.hasOwnProperty.call(detector, 'executionBackend'); + if (!hasIdentity && !target.legacy) { + throw new Error('detector report is missing executionBackend for a schema-v2 target'); + } + const executionBackend = hasIdentity + ? assertExecutionBackend(detector.executionBackend, 'detector report.executionBackend') + : 'standard'; + if (executionBackend !== target.executionBackend) { + throw new Error( + `detector report executionBackend "${executionBackend}" does not match target ` + + `"${target.executionBackend}"` + ); + } + if (!target.legacy && detector.schemaVersion !== 2) { + throw new Error( + `detector report schemaVersion ${JSON.stringify(detector.schemaVersion)} does not match ` + + 'schema-v2 target' + ); + } + for (const field of ['engineVersion', 'grammarHash']) { + const hasField = Object.prototype.hasOwnProperty.call(detector, field); + if (!hasField && !target.legacy) { + throw new Error(`detector report is missing ${field} for a schema-v2 target`); + } + if (hasField && detector[field] !== target[field]) { + throw new Error( + `detector report ${field} ${JSON.stringify(detector[field])} does not match target ` + + `${JSON.stringify(target[field])}` + ); + } + } + if (!Array.isArray(detector.results)) { + throw new TypeError('detector report.results must be a JSON array'); + } + if (!['runtime-bundle', 'compiled-simplified'].includes(detector.surface)) { + throw new Error( + `detector report.surface must be "runtime-bundle" or "compiled-simplified", got ` + + `${JSON.stringify(detector.surface)}` + ); + } + if (!Array.isArray(detector.defaultErrorRules)) { + throw new TypeError('detector report.defaultErrorRules must be a JSON array'); + } + const census = new Set(); + for (const ruleId of detector.defaultErrorRules) { + if (typeof ruleId !== 'string' || ruleId.length === 0) { + throw new TypeError('detector report.defaultErrorRules entries must be non-empty strings'); + } + if (census.has(ruleId)) { + throw new Error(`detector report.defaultErrorRules contains duplicate rule "${ruleId}"`); + } + census.add(ruleId); + } + + const results = new Map(); + for (const entry of detector.results) { + const key = reportRowKey(entry, 'detector report'); + rowExecutionBackend(entry, target, 'detector report', key); + validateOptionalRowIdentity(entry, target, 'detector report', key); + if (results.has(key)) { + throw new Error(`duplicate detector report key "${key}"`); + } + if (!entry.notApplicable && entry.outcome !== 'not-applicable') { + if (!Number.isInteger(entry.expected) || entry.expected < 0) { + throw new TypeError(`detector report row ${key}.expected must be a non-negative integer`); + } + if (!Number.isInteger(entry.actual) || entry.actual < 0) { + throw new TypeError(`detector report row ${key}.actual must be a non-negative integer`); + } + if (!Array.isArray(entry.severities)) { + throw new TypeError(`detector report row ${key}.severities must be a JSON array`); + } + if (typeof entry.severityMatched !== 'boolean') { + throw new TypeError(`detector report row ${key}.severityMatched must be a boolean`); + } + if (typeof entry.messageMatched !== 'boolean') { + throw new TypeError(`detector report row ${key}.messageMatched must be a boolean`); + } + } + results.set(key, entry); + } + return { ...detector, executionBackend, resultsByKey: results }; +} + +function makeLegKey({ label, version, surface, executionBackend }) { + return [label, version, surface, executionBackend] + .map((part) => encodeURIComponent(part)) + .join('::'); +} + +function legFields(leg) { + return { + version: leg.version, + leg: leg.label, + legKey: leg.key, + executionBackend: leg.executionBackend, + }; +} + +function findingKey(finding) { + const backend = Array.isArray(finding.executionBackends) + ? finding.executionBackends.join('-vs-') + : finding.executionBackend || 'standard'; + return [ + finding.legKey || finding.leg || finding.version, + backend, + finding.ruleId, + finding.queryName || '', + finding.driftClass, + ] + .map((part) => encodeURIComponent(String(part))) + .join('::'); +} + +function reportItemKey(item, kind) { + return [ + item.legKey || item.leg || item.version, + item.executionBackend || 'standard', + item.ruleId, + item.queryName || '', + kind, + ] + .map((part) => encodeURIComponent(String(part))) + .join('::'); +} + /** Load the contract corpus, keyed by ruleId, plus the manifest's enforced sets. */ function loadContracts(dir) { const manifest = readJson(path.join(dir, 'manifest.json')); const specs = new Map(); for (const name of manifest.contracts || []) { const spec = readJson(path.join(dir, name)); + try { + assertContractSchema(spec); + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new TypeError(`[${spec.ruleId}] expectations must be a non-empty array`); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + for (const queryExpectation of Object.values(expectation.queries)) { + resolveBackendOracle(spec, queryExpectation, 'standard'); + resolveBackendOracle(spec, queryExpectation, 'analytics'); + } + } + } catch (error) { + artifactFatal(path.join(dir, name), error); + } specs.set(spec.ruleId, { spec, file: name }); } // `defaultError` is the multi-version enforced set: every rule that ships @@ -125,14 +338,45 @@ function loadContracts(dir) { * exists to prevent. */ function loadLeg({ version, dir }) { - const target = readJson(path.join(dir, 'target.json')); - const detector = readJson(path.join(dir, 'detector-report.json')); + const targetFile = path.join(dir, 'target.json'); + const detectorFile = path.join(dir, 'detector-report.json'); + const backendFile = path.join(dir, 'backend-report.json'); + const targetRaw = readJson(targetFile); + const detectorRaw = readJson(detectorFile); const backendRaw = readJson(path.join(dir, 'backend-report.json')); const bundle = readJson(path.join(dir, 'ppl-grammar-bundle.json'), { optional: true }); - const backend = new Map(); - for (const entry of Array.isArray(backendRaw) ? backendRaw : []) { - backend.set(`${entry.ruleId}::${entry.queryName}`, entry); + let target; + let detector; + let backend; + try { + target = normalizeTarget(targetRaw); + } catch (error) { + artifactFatal(targetFile, error); + } + try { + detector = normalizeDetectorReport(detectorRaw, target); + } catch (error) { + artifactFatal(detectorFile, error); + } + try { + backend = indexBackendReport(backendRaw, target); + for (const [key, entry] of backend) { + validateOptionalRowIdentity(entry, target, 'backend report', key); + } + } catch (error) { + artifactFatal(backendFile, error); + } + if ( + bundle && + Object.prototype.hasOwnProperty.call(bundle, 'grammarHash') && + bundle.grammarHash !== target.grammarHash + ) { + fatal( + `grammar bundle ${path.join(dir, 'ppl-grammar-bundle.json')} reports ` + + `${JSON.stringify(bundle.grammarHash)} but target reports ` + + `${JSON.stringify(target.grammarHash)}` + ); } // The engine's self-reported version wins over the matrix label, so a matrix @@ -145,11 +389,15 @@ function loadLeg({ version, dir }) { ); } - return { + const leg = { version: reported || version, label: version, dir, grammarHash: target.grammarHash || '', + sqlSha: target.sqlSha || '', + executionBackend: target.executionBackend, + targetSchemaVersion: target.schemaVersion, + legacyTarget: target.legacy, // Which of OSD's two lint surfaces this leg validated. Older detector reports // predate the field; they were all runtime-bundle runs. surface: detector.surface || 'runtime-bundle', @@ -157,6 +405,221 @@ function loadLeg({ version, dir }) { detector, backend, }; + leg.key = makeLegKey(leg); + return leg; +} + +function pairBackendLegs(legs) { + const identities = new Set(); + for (const leg of legs) { + if (identities.has(leg.key)) { + fatal(`duplicate leg identity "${leg.key}"`); + } + identities.add(leg.key); + } + + const runtimeLegs = legs.filter((leg) => leg.surface === 'runtime-bundle'); + const standards = runtimeLegs.filter((leg) => leg.executionBackend === 'standard'); + const analyticsLegs = runtimeLegs.filter((leg) => leg.executionBackend === 'analytics'); + const usedStandards = new Set(); + const pairs = []; + const neutralLabel = (label) => String(label).replace(/[-_](?:standard|analytics)$/i, ''); + + for (const analytics of analyticsLegs) { + const labelPeers = standards.filter( + (standard) => neutralLabel(standard.label) === neutralLabel(analytics.label) + ); + const candidates = + labelPeers.length > 0 + ? labelPeers + : standards.filter((standard) => standard.version === analytics.version); + if (candidates.length === 0) { + if (standards.length > 0) { + fatal( + `analytics leg "${analytics.key}" has no standard peer for engine ` + + `${analytics.version}` + ); + } + continue; + } + + const sameLabel = candidates.filter((standard) => standard.label === analytics.label); + const sameGrammar = candidates.filter( + (standard) => standard.grammarHash === analytics.grammarHash + ); + let standard; + if (sameLabel.length === 1) { + standard = sameLabel[0]; + } else if (sameGrammar.length === 1) { + standard = sameGrammar[0]; + } else if (candidates.length === 1) { + standard = candidates[0]; + } else { + fatal( + `analytics leg "${analytics.key}" has ${candidates.length} possible standard peers for ` + + `${analytics.version}; use an unambiguous label/grammar identity` + ); + } + + if (standard.version !== analytics.version) { + fatal( + `paired standard/analytics legs report different engine versions: ` + + `${standard.label}=${JSON.stringify(standard.version)}, ` + + `${analytics.label}=${JSON.stringify(analytics.version)}` + ); + } + if (!standard.sqlSha || !analytics.sqlSha) { + fatal( + `paired standard/analytics legs must both report a non-empty SQL SHA: ` + + `${standard.label}=${JSON.stringify(standard.sqlSha)}, ` + + `${analytics.label}=${JSON.stringify(analytics.sqlSha)}` + ); + } + if (standard.sqlSha !== analytics.sqlSha) { + fatal( + `paired standard/analytics legs report different SQL SHAs: ` + + `${standard.label}=${JSON.stringify(standard.sqlSha)}, ` + + `${analytics.label}=${JSON.stringify(analytics.sqlSha)}` + ); + } + if (!standard.grammarHash || !analytics.grammarHash) { + fatal( + `paired standard/analytics legs for ${analytics.version} must both report a runtime grammar hash` + ); + } + if (standard.grammarHash !== analytics.grammarHash) { + fatal( + `paired standard/analytics legs for ${analytics.version} have different grammar hashes: ` + + `${standard.label}=${JSON.stringify(standard.grammarHash)}, ` + + `${analytics.label}=${JSON.stringify(analytics.grammarHash)}` + ); + } + if (usedStandards.has(standard.key)) { + fatal( + `standard leg "${standard.key}" matches more than one analytics leg; duplicate backend leg identity` + ); + } + usedStandards.add(standard.key); + const pair = { + key: `${standard.key}::${analytics.key}`, + standard, + analytics, + engineVersion: analytics.version, + grammarHash: analytics.grammarHash, + }; + assertDetectorParity(pair); + pairs.push(pair); + } + return pairs; +} + +function detectorParityValue(entry) { + return { + role: entry.role || 'trigger', + query: entry.query || '', + expected: entry.expected, + actual: entry.actual, + severities: [...(entry.severities || [])].sort(), + severityMatched: + typeof entry.severityMatched === 'boolean' ? entry.severityMatched : undefined, + messageMatched: + typeof entry.messageMatched === 'boolean' ? entry.messageMatched : undefined, + }; +} + +/** + * Both detector passes use the same OSD checkout, grammar, contracts, and lint + * context. Any route-qualified difference is therefore a harness defect, not a + * backend observation. + */ +function assertDetectorParity(pair) { + const standard = pair.standard.detector.resultsByKey; + const analytics = pair.analytics.detector.resultsByKey; + const keys = new Set([...standard.keys(), ...analytics.keys()]); + for (const key of keys) { + const standardRow = standard.get(key); + const analyticsRow = analytics.get(key); + if (!standardRow || !analyticsRow) { + fatal( + `detector parity failed for ${key}: standard row=${!!standardRow}, ` + + `analytics row=${!!analyticsRow}` + ); + } + const standardValue = detectorParityValue(standardRow); + const analyticsValue = detectorParityValue(analyticsRow); + if (JSON.stringify(standardValue) !== JSON.stringify(analyticsValue)) { + fatal( + `detector parity failed for ${key}: standard=${JSON.stringify(standardValue)}, ` + + `analytics=${JSON.stringify(analyticsValue)}` + ); + } + } +} + +function backendVerdict(entry) { + if (!entry) { + return { + backendRejected: undefined, + backendType: undefined, + backendReason: undefined, + }; + } + const state = classifyBackendReportRow(entry); + const observedBackend = entry && entry.observed; + const rowRejected = + typeof entry.rejected === 'boolean' ? entry.rejected : undefined; + const observedRejected = + observedBackend && typeof observedBackend.rejected === 'boolean' + ? observedBackend.rejected + : undefined; + if ( + typeof rowRejected === 'boolean' && + typeof observedRejected === 'boolean' && + rowRejected !== observedRejected + ) { + fatal( + `backend report row ${reportRowKey(entry, 'backend report')} has conflicting ` + + `rejected verdicts` + ); + } + const explicitRejected = + typeof observedRejected === 'boolean' ? observedRejected : rowRejected; + const usableRawObservation = + state.status === 'observed' || state.status === 'coverage-missing'; + return { + backendRejected: + usableRawObservation && typeof explicitRejected === 'boolean' + ? explicitRejected + : undefined, + backendStatus: observedBackend ? observedBackend.httpStatus : undefined, + backendType: observedBackend ? observedBackend.type : undefined, + backendReason: observedBackend ? observedBackend.reason : undefined, + backendOutcome: entry.outcome, + backendMismatch: entry.error, + }; +} + +function indexDivergentCases(pairs) { + const cases = new Map(); + for (const pair of pairs) { + for (const [rowKey, standardEntry] of pair.standard.backend) { + const analyticsEntry = pair.analytics.backend.get(rowKey); + if (!analyticsEntry) continue; + const standardObserved = backendVerdict(standardEntry); + const analyticsObserved = backendVerdict(analyticsEntry); + if ( + typeof standardObserved.backendRejected !== 'boolean' || + typeof analyticsObserved.backendRejected !== 'boolean' || + standardObserved.backendRejected === analyticsObserved.backendRejected + ) { + continue; + } + const value = { pair, rowKey, standardObserved, analyticsObserved }; + cases.set(`${pair.standard.key}::${rowKey}`, value); + cases.set(`${pair.analytics.key}::${rowKey}`, value); + } + } + return cases; } /** @@ -216,23 +679,22 @@ function auditDefaultErrorCensus(legs, specs, enforcedRules) { * comparable engine verdict, so the caller can refuse to call it agreement. */ function readBackendObservation(backendEntry, detectorResult) { - const observedBackend = (backendEntry && backendEntry.observed) || undefined; - const outcome = backendEntry && backendEntry.outcome; - // `observed`/`error` are the observe-only outcomes; `pass`/`fail` come from the - // asserting mode. Only those carry a real verdict. - const hasVerdict = - !!backendEntry && - outcome !== 'error' && - (typeof backendEntry.rejected === 'boolean' || !!observedBackend); + const verdict = backendVerdict(backendEntry); + const hasVerdict = typeof verdict.backendRejected === 'boolean'; return { usable: hasVerdict && !!detectorResult, observed: { detectorCount: detectorResult ? detectorResult.actual : 0, severities: detectorResult ? detectorResult.severities || [] : [], - backendRejected: hasVerdict ? !!backendEntry.rejected : undefined, - backendType: observedBackend ? observedBackend.type : undefined, - backendReason: observedBackend ? observedBackend.reason : undefined, + backendRejected: verdict.backendRejected, + backendStatus: verdict.backendStatus, + backendType: verdict.backendType, + backendReason: verdict.backendReason, + backendOutcome: verdict.backendOutcome, + backendMismatch: verdict.backendMismatch, + severityMatched: detectorResult ? detectorResult.severityMatched : undefined, + messageMatched: detectorResult ? detectorResult.messageMatched : undefined, }, }; } @@ -247,28 +709,31 @@ function readBackendObservation(backendEntry, detectorResult) { * expectation to read on this path), and the backend observation from this leg's * report; `classifyDrift` decides, so the "too narrow" wording stays in one place. */ -function classifyOutOfScope({ spec, ruleId, leg, classify }) { +function classifyOutOfScope({ spec, ruleId, leg, classify, divergentCases }) { const found = []; + const unusable = []; + const observations = new Map(); + + for (const [queryName] of Object.entries(spec.queries || {})) { + const rowKey = `${ruleId}::${queryName}`; + const backendEntry = leg.backend.get(rowKey); + const detectorResult = leg.detector.resultsByKey.get(rowKey); + const { observed, usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + unusable.push( + `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` + ); + continue; + } + observations.set(queryName, observed); + } // What did this rule's CONTROL queries — valid uses of the same command — do on // this engine? THREE states, not two, and the difference decides whether a - // rejected trigger means anything: - // rejected the command itself is unsupported here, so the trigger's rejection - // says nothing about the rule's specific condition -> suppress - // accepted the command works, so a rejected trigger really is the rule's - // condition going unreported on this version -> report it - // unknown no control verdict arrived (errored/absent). We cannot tell the two - // apart, so we must not emit confident advice either way. - // Collapsing this to a boolean is what let the suppression fail open: an errored - // control read as "not rejected" and produced the exact "widen appliesTo" advice - // this check exists to prevent. + // rejected trigger means anything. const controlVerdicts = Object.entries(spec.queries || {}) .filter(([, def]) => (def.role || 'trigger') === 'control') - .map(([name]) => { - const entry = leg.backend.get(`${ruleId}::${name}`); - const { observed } = readBackendObservation(entry, { actual: 0, severities: [] }); - return observed.backendRejected; - }); + .map(([name]) => observations.get(name)?.backendRejected); const controlAlsoRejected = controlVerdicts.some((v) => v === true); // A rule with controls, none of which produced a verdict, cannot be judged here. const controlUnknown = @@ -276,16 +741,11 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { if ((queryDef.role || 'trigger') !== 'trigger') continue; - const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); - if (!backendEntry) continue; // this leg never ran the query - const detectorResult = (leg.detector.results || []).find( - (r) => r.ruleId === ruleId && r.queryName === queryName - ); - // Same reason as above: an errored observation must not read as "the engine - // accepted this". On this path that coercion would turn a genuinely - // mis-scoped rule into a silent `out-of-scope` PASS, because the - // version-scope-too-narrow check requires backendRejected === true. - const { observed: outOfScopeObserved } = readBackendObservation(backendEntry, detectorResult); + const rowKey = `${ruleId}::${queryName}`; + const outOfScopeObserved = observations.get(queryName); + if (!outOfScopeObserved) continue; + const pairedDivergence = divergentCases.has(`${leg.key}::${rowKey}`); + if (pairedDivergence && leg.executionBackend === 'analytics') continue; const drift = classify({ ruleId, version: leg.version, @@ -297,6 +757,7 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { observed: outOfScopeObserved, wiring: spec.wiring, detectorPath: spec.detectorPath, + executionBackend: leg.executionBackend, // An unknown control verdict is treated the same as a rejected one: both // mean "we cannot claim this engine supports the command", and staying quiet // is the only honest option. @@ -306,7 +767,7 @@ function classifyOutOfScope({ spec, ruleId, leg, classify }) { }); if (drift) found.push(drift); } - return found; + return { drifts: found, unusable }; } /** @@ -376,11 +837,14 @@ function main() { const versionMatchesRange = makeRangeMatcher(); const { specs, enforcedRules, manifest } = loadContracts(args.contracts); const legs = args.legs.map(loadLeg); + const backendPairs = pairBackendLegs(legs); + const divergentCases = indexDivergentCases(backendPairs); log(`contracts=${specs.size} enforced(default-error)=${enforcedRules.size} legs=${legs.length}`); for (const leg of legs) { log( - ` leg ${leg.label}: engine=${leg.version} grammar=${(leg.grammarHash || '—').slice(0, 19)} ` + + ` leg ${leg.label} (${leg.executionBackend}): engine=${leg.version} ` + + `grammar=${(leg.grammarHash || '—').slice(0, 19)} ` + `detectorResults=${(leg.detector.results || []).length} backendCases=${leg.backend.size}` ); } @@ -395,7 +859,18 @@ function main() { // compiled-simplified leg). Recorded so the report can say WHY a cell is blank, // but never a failure: the rule is inert there by design. const notApplicable = []; - const matrix = []; // one row per rule × version, for the summary table + const matrix = []; // one row per rule × backend-qualified leg, for the summary table + const addDrift = (drift, leg, extra = {}) => { + const enriched = { + ...drift, + ...legFields(leg), + ...extra, + executionBackend: drift.executionBackend || leg.executionBackend, + }; + enriched.key = findingKey(enriched); + drifts.push(enriched); + return enriched; + }; // A rule that ships enabled at error severity but has no contract file is // invisible to this whole check. Compare the manifest's declared set against @@ -425,15 +900,13 @@ function main() { if (contractSurface !== 'both' && contractSurface !== legSurface) { notApplicable.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), surface: legSurface, reason: `contract declares grammarSurface "${contractSurface}"`, }); matrix.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), status: 'not-applicable', drifts: 0, }); @@ -453,10 +926,11 @@ function main() { requiredParserRules: spec.requiredParserRules, detectorPath: spec.detectorPath, parserRuleNames: leg.parserRuleNames, + executionBackend: leg.executionBackend, }); if (grammarDrift) { - drifts.push({ ...grammarDrift, enforced: isEnforced, contractFile: file }); - matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'drift', drifts: 1 }); + addDrift(grammarDrift, leg, { enforced: isEnforced, contractFile: file }); + matrix.push({ ruleId, ...legFields(leg), status: 'drift', drifts: 1 }); continue; } } @@ -467,27 +941,47 @@ function main() { // Deliberately out of scope on this engine. Still run the classifier // for the one case that matters — an engine that rejects a trigger the // rule has been scoped away from (a missed diagnostic). - const outOfScopeDrifts = classifyOutOfScope({ + const outOfScope = classifyOutOfScope({ spec, ruleId, leg, classify: classifyDrift, + divergentCases, }); - for (const drift of outOfScopeDrifts) { - drifts.push({ ...drift, enforced: isEnforced, contractFile: file }); + for (const drift of outOfScope.drifts) { + addDrift(drift, leg, { enforced: isEnforced, contractFile: file }); + } + if (outOfScope.unusable.length > 0) { + inconclusive.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reasons: outOfScope.unusable, + }); } matrix.push({ ruleId, - version: leg.version, - leg: leg.label, - status: outOfScopeDrifts.length > 0 ? 'drift' : 'out-of-scope', - drifts: outOfScopeDrifts.length, + ...legFields(leg), + status: + outOfScope.unusable.length > 0 + ? 'inconclusive' + : outOfScope.drifts.length > 0 + ? 'drift' + : 'out-of-scope', + drifts: outOfScope.drifts.length, }); continue; } // In scope on this engine but nothing pins its behavior there. - coverageHoles.push({ ruleId, file, version: leg.version, enforced: isEnforced }); - matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'uncovered', drifts: 0 }); + coverageHoles.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reason: 'no version expectation matches this engine', + }); + matrix.push({ ruleId, ...legFields(leg), status: 'uncovered', drifts: 0 }); continue; } @@ -505,6 +999,7 @@ function main() { // because the two need opposite advice: not-applicable is expected and needs // no action, unusable means something did not answer and needs a re-run. let ruleNotApplicable = 0; + let ruleCoverageHoles = 0; const unusable = []; // Per-trigger engine verdicts for this rule on this leg, so a relaxation can // be judged across the WHOLE rule rather than one query at a time. A single @@ -534,9 +1029,99 @@ function main() { triggersExpected++; } - const detectorResult = (leg.detector.results || []).find( - (r) => r.ruleId === ruleId && r.queryName === queryName - ); + let oracleSelection; + try { + oracleSelection = resolveBackendOracle(spec, expected, leg.executionBackend); + } catch (error) { + artifactFatal(`${file} query "${queryName}"`, error); + } + const rowKey = `${ruleId}::${queryName}`; + const detectorResult = leg.detector.resultsByKey.get(rowKey); + const backendEntry = leg.backend.get(rowKey); + if ( + detectorResult && + !detectorResult.notApplicable && + detectorResult.outcome !== 'not-applicable' + ) { + if (detectorResult.expected !== oracleSelection.detector.count) { + fatal( + `detector report row ${rowKey} expected=${JSON.stringify(detectorResult.expected)} ` + + `does not match contract detectorCount=${oracleSelection.detector.count}` + ); + } + if ((detectorResult.role || 'trigger') !== role) { + fatal( + `detector report row ${rowKey} role=${JSON.stringify(detectorResult.role)} ` + + `does not match contract role=${JSON.stringify(role)}` + ); + } + } + + if (oracleSelection.status === 'coverage-missing') { + const { usable } = readBackendObservation(backendEntry, detectorResult); + if (!usable) { + unusable.push( + `${queryName} (${!detectorResult ? 'no detector result' : 'no engine verdict'})` + ); + if (role === 'trigger') { + unobservedTriggers.push(queryName); + } + continue; + } + coverageHoles.push({ + ruleId, + queryName, + file, + ...legFields(leg), + enforced: isEnforced, + reason: oracleSelection.reason, + kind: 'backend-oracle', + }); + ruleCoverageHoles++; + continue; + } + if (oracleSelection.status === 'not-applicable') { + const backendState = backendEntry + ? classifyBackendReportRow(backendEntry) + : { status: 'error' }; + if (!detectorResult || backendState.status !== 'not-applicable') { + unusable.push( + `${queryName} (${ + !detectorResult + ? 'no detector result' + : 'backend did not report not-applicable' + })` + ); + continue; + } + notApplicable.push({ + ruleId, + queryName, + ...legFields(leg), + surface: leg.surface, + reason: oracleSelection.reason, + kind: 'backend-oracle', + }); + ruleNotApplicable++; + if (isEnforced) { + coverageHoles.push({ + ruleId, + queryName, + file, + ...legFields(leg), + enforced: true, + reason: + `default-error rule is not applicable on ${leg.executionBackend}: ` + + `${oracleSelection.reason}`, + kind: 'backend-oracle', + issue: oracleSelection.oracle.issue, + owner: oracleSelection.oracle.owner, + }); + ruleCoverageHoles++; + } + continue; + } + // A case the surface cannot express at all (a `runtimeOnly` rule on a // compiled-simplified leg) is excluded rather than compared. Its zero // diagnostics are `lint_runner` deliberately skipping the rule, so @@ -546,15 +1131,14 @@ function main() { if (detectorResult && detectorResult.notApplicable) { notApplicable.push({ ruleId, - version: leg.version, queryName, + ...legFields(leg), surface: leg.surface, reason: detectorResult.notApplicable, }); ruleNotApplicable++; continue; } - const backendEntry = leg.backend.get(`${ruleId}::${queryName}`); const { observed, usable } = readBackendObservation(backendEntry, detectorResult); if (!usable) { // No comparable pair, so there is nothing to classify. Attempting it @@ -571,6 +1155,7 @@ function main() { continue; } compared++; + const pairedDivergence = divergentCases.get(`${leg.key}::${rowKey}`); if (role === 'trigger') { triggersCompared++; // Bucket this trigger by what the ENGINE did, but only where the contract @@ -578,8 +1163,11 @@ function main() { // head-without-sort, whose queries are all valid PPL) never "relaxes", and // counting it as relaxed would fabricate a full-fix verdict for a rule the // engine was never rejecting in the first place. - const pinnedRejection = (expected.backend && expected.backend.kind) === 'rejection'; - if (pinnedRejection) { + const pinnedRejection = oracleSelection.oracle.kind === 'rejection'; + if ( + pinnedRejection && + (!pairedDivergence || leg.executionBackend === 'standard') + ) { if (observed.backendRejected === false) { relaxedTriggers.push(queryName); if ((observed.detectorCount || 0) > 0) relaxedDetectorFlagged = true; @@ -589,24 +1177,29 @@ function main() { } } - const drift = classifyDrift({ - ruleId, - version: leg.version, - queryName, - role, - query, - expected: { - detectorCount: expected.detectorCount, - severity: expected.severity, - backendKind: expected.backend && expected.backend.kind, - }, - observed, - wiring: spec.wiring, - detectorPath: spec.detectorPath, - parserRuleNames: leg.parserRuleNames, - requiredParserRules: spec.requiredParserRules, - expectedBackend: expected.backend, - }); + const drift = + pairedDivergence && leg.executionBackend === 'analytics' + ? null + : classifyDrift({ + ruleId, + version: leg.version, + queryName, + role, + query, + expected: { + detectorCount: oracleSelection.detector.count, + severity: oracleSelection.detector.severity, + matchMessage: oracleSelection.detector.matchMessage, + backendKind: oracleSelection.oracle.kind, + }, + observed, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + parserRuleNames: leg.parserRuleNames, + requiredParserRules: spec.requiredParserRules, + expectedBackend: oracleSelection.oracle, + executionBackend: leg.executionBackend, + }); if (drift) { // `expectationRange` is what the annotation anchors to: the version @@ -630,26 +1223,29 @@ function main() { // rule as a whole. This supersedes the per-query `engine-relaxed` findings — // they each said "scope this rule away from this version", which is the wrong // action whenever another trigger still rejects. - const relaxationScope = classifyRelaxationScope({ - ruleId, - version: leg.version, - relaxedTriggers, - holdingTriggers, - unobservedTriggers, - detectorFlagged: relaxedDetectorFlagged, - wiring: spec.wiring, - detectorPath: spec.detectorPath, - }); + const relaxationScope = + leg.executionBackend === 'standard' + ? classifyRelaxationScope({ + ruleId, + version: leg.version, + relaxedTriggers, + holdingTriggers, + unobservedTriggers, + detectorFlagged: relaxedDetectorFlagged, + wiring: spec.wiring, + detectorPath: spec.detectorPath, + executionBackend: leg.executionBackend, + }) + : null; const kept = relaxationScope ? perQueryDrifts.filter((d) => d.supersededBy !== DRIFT_CLASSES.ENGINE_PARTIALLY_RELAXED) : perQueryDrifts; for (const drift of kept) { - drifts.push(drift); + addDrift(drift, leg); ruleDrifts++; } if (relaxationScope) { - drifts.push({ - ...relaxationScope, + addDrift(relaxationScope, leg, { enforced: isEnforced, contractFile: file, expectationRange: expectation.version, @@ -667,11 +1263,31 @@ function main() { // nothing failed and there is nothing to re-run, so it must not fail the run. // Checked BEFORE the inconclusive test, which would otherwise catch it // (compared === 0) and demand a re-run that could never change the outcome. - if (compared === 0 && ruleNotApplicable > 0) { + if (unusable.length > 0) { + inconclusive.push({ + ruleId, + file, + ...legFields(leg), + enforced: isEnforced, + reasons: unusable, + }); matrix.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), + status: 'inconclusive', + drifts: ruleDrifts, + }); + } else if (ruleCoverageHoles > 0) { + matrix.push({ + ruleId, + ...legFields(leg), + status: ruleDrifts > 0 ? 'drift' : 'uncovered', + drifts: ruleDrifts, + }); + } else if (compared === 0 && ruleNotApplicable > 0) { + matrix.push({ + ruleId, + ...legFields(leg), status: 'not-applicable', drifts: 0, }); @@ -679,40 +1295,125 @@ function main() { inconclusive.push({ ruleId, file, - version: leg.version, + ...legFields(leg), enforced: isEnforced, reasons: unusable, }); - matrix.push({ ruleId, version: leg.version, leg: leg.label, status: 'inconclusive', drifts: ruleDrifts }); + matrix.push({ + ruleId, + ...legFields(leg), + status: 'inconclusive', + drifts: ruleDrifts, + }); } else { - if (unusable.length > 0) { - log( - `WARN: ${ruleId} @ ${leg.version} compared ${compared} case(s); ` + - `${unusable.length} not compared: ${unusable.join(', ')}` - ); - } matrix.push({ ruleId, - version: leg.version, - leg: leg.label, + ...legFields(leg), status: ruleDrifts === 0 ? 'agree' : 'drift', drifts: ruleDrifts, }); } } + + const contractSurface = spec.grammarSurface || 'runtime-bundle'; + if (contractSurface === 'runtime-bundle' || contractSurface === 'both') { + for (const pair of backendPairs) { + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + const rowKey = `${ruleId}::${queryName}`; + const divergent = divergentCases.get(`${pair.analytics.key}::${rowKey}`); + if (!divergent || divergent.pair.key !== pair.key) continue; + + const query = queryDef.query.split('{{index}}').join(spec.index); + const drift = classifyExecutionBackendDivergence({ + ruleId, + version: pair.engineVersion, + queryName, + role: queryDef.role || 'trigger', + query, + standardObserved: divergent.standardObserved, + analyticsObserved: divergent.analyticsObserved, + standardLeg: pair.standard.label, + analyticsLeg: pair.analytics.label, + grammarHash: pair.grammarHash, + detectorPath: spec.detectorPath, + }); + if (!drift) continue; + + const expectation = selectExpectation(spec, pair.engineVersion, versionMatchesRange); + addDrift(drift, pair.analytics, { + enforced: isEnforced, + contractFile: file, + expectationRange: expectation && expectation.version, + expectationEngine: expectation && expectation.engine, + pairKey: pair.key, + standardLeg: pair.standard.label, + standardLegKey: pair.standard.key, + analyticsLeg: pair.analytics.label, + analyticsLegKey: pair.analytics.key, + }); + + for (const row of matrix) { + if ( + row.ruleId === ruleId && + (row.legKey === pair.standard.key || row.legKey === pair.analytics.key) + ) { + row.status = 'drift'; + row.drifts += 1; + } + } + } + } + } } - const enforcedDrifts = drifts.filter((d) => d.enforced); - const enforcedHoles = coverageHoles.filter((h) => h.enforced); + const isObservedAnalyticsFinding = (entry) => + args.observeAnalytics && + (entry.executionBackend === 'analytics' || + (Array.isArray(entry.executionBackends) && + entry.executionBackends.includes('analytics'))) && + (entry.driftClass === DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE || + entry.driftClass === DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH || + entry.kind === 'backend-oracle'); + for (const drift of drifts) { + drift.blocking = !!drift.enforced && !isObservedAnalyticsFinding(drift); + } + for (const hole of coverageHoles) { + hole.blocking = !!hole.enforced && !isObservedAnalyticsFinding(hole); + } + const enforcedDrifts = drifts.filter((d) => d.blocking); + const enforcedHoles = coverageHoles.filter((h) => h.blocking); const enforcedInconclusive = inconclusive.filter((i) => i.enforced); + for (const row of matrix) { + row.key = reportItemKey(row, 'matrix'); + } + for (const hole of coverageHoles) { + hole.key = reportItemKey(hole, 'coverage-hole'); + } + for (const entry of inconclusive) { + entry.key = reportItemKey(entry, 'inconclusive'); + } + for (const entry of notApplicable) { + entry.key = reportItemKey(entry, 'not-applicable'); + } const report = { - schemaVersion: 1, + schemaVersion: 2, + keyDimensions: ['leg', 'engineVersion', 'grammarSurface', 'executionBackend'], legs: legs.map((l) => ({ + key: l.key, label: l.label, engineVersion: l.version, grammarHash: l.grammarHash, + sqlSha: l.sqlSha, surface: l.surface, + executionBackend: l.executionBackend, + })), + backendPairs: backendPairs.map((pair) => ({ + key: pair.key, + engineVersion: pair.engineVersion, + grammarHash: pair.grammarHash, + standardLegKey: pair.standard.key, + analyticsLegKey: pair.analytics.key, })), enforcedRules: [...enforcedRules].sort(), missingContracts, @@ -726,6 +1427,9 @@ function main() { driftCount: drifts.length, enforcedDriftCount: enforcedDrifts.length, enforcedCoverageHoles: enforcedHoles.length, + observedAnalyticsFindings: + drifts.filter((d) => d.enforced && !d.blocking).length + + coverageHoles.filter((h) => h.enforced && !h.blocking).length, missingContractCount: missingContracts.length, enforcedInconclusive: enforcedInconclusive.length, // An inconclusive default-error rule fails too: "we could not check" must @@ -792,6 +1496,9 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { `${report.result.enforcedDriftCount} enforced drift(s)`, `${report.result.enforcedCoverageHoles} coverage hole(s)`, ]; + if (report.result.observedAnalyticsFindings) { + reasons.push(`${report.result.observedAnalyticsFindings} analytics observation(s)`); + } if (report.result.enforcedInconclusive) { reasons.push(`${report.result.enforcedInconclusive} inconclusive`); } @@ -803,23 +1510,28 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { // reader knows a column speaks for OSD's compiled grammar rather than the // engine's exported one — the two do not run the same set of rules. `Engine versions: ${legs - .map((l) => - l.surface && l.surface !== 'runtime-bundle' ? `\`${l.version}\` (${l.surface})` : `\`${l.version}\`` - ) + .map((l) => { + const identity = + l.label === l.version ? `\`${l.version}\`` : `\`${l.label}\` → \`${l.version}\``; + return l.surface && l.surface !== 'runtime-bundle' + ? `${identity} (${l.executionBackend}, ${l.surface})` + : `${identity} (${l.executionBackend})`; + }) .join(', ')} — ` + `**${report.result.passed ? 'PASS' : 'FAIL'}** (${reasons.join(', ')})` ); lines.push(''); - // Columns are keyed on the LEG LABEL, not the engine version: two legs can share - // a version while validating different surfaces (a 3.7 runtime-bundle leg and a - // 3.7 compiled leg), and keying on version alone made them collide so one leg's - // results silently rendered in place of the other's. + // Columns use the full leg key, including execution backend and grammar surface. + // A label or engine version alone is not unique once the same candidate runs + // through both standard and analytics. const columns = legs.map((l) => ({ - label: l.label, + key: l.key, heading: l.surface && l.surface !== 'runtime-bundle' - ? `\`${l.version}\`
      ${l.surface}` - : `\`${l.version}\``, + ? `\`${l.version}\`
      ${l.executionBackend}
      ${l.surface}` + + (l.label === l.version ? '' : `
      ${l.label}`) + : `\`${l.version}\`
      ${l.executionBackend}` + + (l.label === l.version ? '' : `
      ${l.label}`), })); const rules = [...new Set(report.matrix.map((m) => m.ruleId))].sort(); lines.push(`| Rule | ${columns.map((c) => c.heading).join(' | ')} |`); @@ -834,7 +1546,7 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { }; for (const ruleId of rules) { const cells = columns.map((column) => { - const row = report.matrix.find((m) => m.ruleId === ruleId && m.leg === column.label); + const row = report.matrix.find((m) => m.ruleId === ruleId && m.legKey === column.key); if (!row) return '—'; if (row.status === 'drift') return `**DRIFT** (${row.drifts})`; // An unmapped status must still render as something visible. A blank cell @@ -851,7 +1563,8 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { lines.push(''); for (const entry of report.inconclusive) { lines.push( - `- \`${entry.ruleId}\` on engine \`${entry.version}\`: no case could be compared — ` + + `- \`${entry.ruleId}\` on engine \`${entry.version}\` (${entry.executionBackend}): ` + + `no case could be compared — ` + `${entry.reasons.join('; ')}. This is NOT a lint finding: the engine or the detector run ` + `did not answer, so nothing was validated. Check that leg's job logs (an unreachable ` + `cluster, an index that failed to seed, or a detector runner that died mid-corpus) and ` + @@ -880,11 +1593,18 @@ function renderMarkdown(report, drifts, coverageHoles, legs) { lines.push('### Coverage holes'); lines.push(''); for (const hole of coverageHoles) { + const query = hole.queryName ? ` query \`${hole.queryName}\`` : ''; + const fix = + hole.kind === 'backend-oracle' + ? `add a reviewed \`${hole.executionBackend}\` backend oracle for this query` + : `add an \`expectations[]\` entry whose \`version\` range covers \`${hole.version}\`, ` + + `or narrow the rule's \`appliesTo\` so it does not apply there`; lines.push( - `- \`${hole.ruleId}\` has no expectation matching engine \`${hole.version}\`` + + `- \`${hole.ruleId}\`${query} has no ${hole.executionBackend} coverage for engine ` + + `\`${hole.version}\`` + `${hole.enforced ? ' (ENFORCED — this rule ships to users on that engine unpinned)' : ''}. ` + - `FIX (${hole.file}): add an \`expectations[]\` entry whose \`version\` range covers ` + - `\`${hole.version}\`, or narrow the rule's \`appliesTo\` so it does not apply there.` + `${hole.reason ? `${hole.reason}. ` : ''}` + + `FIX (${hole.file}): ${fix}.` ); } lines.push(''); diff --git a/scripts/ppl-lint/annotate.mjs b/scripts/ppl-lint/annotate.mjs index 06b1a384db7..422177ce204 100644 --- a/scripts/ppl-lint/annotate.mjs +++ b/scripts/ppl-lint/annotate.mjs @@ -47,6 +47,12 @@ function escapeData(value) { return String(value).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); } +function backendLabel(entry) { + return Array.isArray(entry.executionBackends) && entry.executionBackends.length > 1 + ? entry.executionBackends.join(' vs ') + : entry.executionBackend || 'standard'; +} + /** * Line of the `expectations[]` entry whose `version` is `range`, 1-indexed. * @@ -124,10 +130,12 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r findRuleIdLine(text); annotations.push({ - level: drift.enforced ? 'error' : 'warning', + level: (drift.blocking ?? drift.enforced) ? 'error' : 'warning', file: file ? contractRepoPath(contractsDir, file, workspace) : undefined, line, - title: `PPL lint drift: ${drift.driftClass} (${drift.ruleId} @ ${drift.version})`, + title: + `PPL lint drift: ${drift.driftClass} ` + + `(${drift.ruleId} @ ${drift.version}, ${backendLabel(drift)})`, // Message order matters: the UI truncates, so lead with what moved, then the // action, then where. The summary carries the full rationale. message: [ @@ -143,15 +151,17 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r for (const hole of report.coverageHoles || []) { const text = contractText(hole.file); annotations.push({ - level: hole.enforced ? 'error' : 'warning', + level: (hole.blocking ?? hole.enforced) ? 'error' : 'warning', file: hole.file ? contractRepoPath(contractsDir, hole.file, workspace) : undefined, line: findRuleIdLine(text), - title: `PPL lint coverage hole: ${hole.ruleId} @ ${hole.version}`, + title: + `PPL lint coverage hole: ${hole.ruleId} @ ${hole.version}, ${backendLabel(hole)}`, message: - `No expectation in this contract matches engine ${hole.version}, so nothing pins ` + - `"${hole.ruleId}" there. Add a reviewed expectation whose version range covers ` + - `${hole.version}; do not widen an existing range to absorb it unless the behavior is ` + - `genuinely identical.`, + (hole.reason + ? `${hole.reason}. ` + : `No expectation in this contract matches engine ${hole.version}. `) + + `Nothing pins "${hole.ruleId}" for the ${backendLabel(hole)} route there. Add a reviewed ` + + `${backendLabel(hole)} oracle or expectation; never use another route's oracle as fallback.`, }); } @@ -163,9 +173,12 @@ export function buildAnnotations(report, { contractsDir, workspace, readFile = r level: 'warning', file: entry.file ? contractRepoPath(contractsDir, entry.file, workspace) : undefined, line: findRuleIdLine(text), - title: `PPL lint inconclusive: ${entry.ruleId} @ ${entry.version} (leg problem)`, + title: + `PPL lint inconclusive: ${entry.ruleId} @ ${entry.version}, ` + + `${backendLabel(entry)} (leg problem)`, message: - `No case could be compared for "${entry.ruleId}" on engine ${entry.version}` + + `No case could be compared for "${entry.ruleId}" on engine ${entry.version} ` + + `(${backendLabel(entry)})` + (entry.reasons && entry.reasons.length > 0 ? ` — ${entry.reasons.join('; ')}` : '') + `. This is NOT a lint finding: the engine or the detector run did not answer, so ` + `nothing was validated. Check that leg's job logs and re-run. Do not edit the rule or ` + diff --git a/scripts/ppl-lint/assemble-run-manifest.mjs b/scripts/ppl-lint/assemble-run-manifest.mjs index 840e631ef88..faf21f078b1 100644 --- a/scripts/ppl-lint/assemble-run-manifest.mjs +++ b/scripts/ppl-lint/assemble-run-manifest.mjs @@ -7,7 +7,7 @@ * Assemble the PPL lint validation run manifest and the compact per-rule PR * summary in the result job (design §3.3, §4.4, T10). * - * Inputs (env, all optional so a partial run still produces a manifest): + * Inputs (env; identity fields may be empty on a partial run): * SQL_SHA, OSD_REF, OSD_SHA, EVENT_NAME, SCHEDULE, * BACKEND_RESULT, DETECTOR_RESULT, GITHUB_STEP_SUMMARY. * Artifact files under ./artifacts (downloaded from both jobs): @@ -22,24 +22,143 @@ import fs from 'fs'; import path from 'path'; +import { + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, +} from './contract-schema.mjs'; + const ARTIFACTS = 'artifacts'; -function readJson(file) { +function readJson(file, errors) { try { if (fs.existsSync(file)) { return JSON.parse(fs.readFileSync(file, 'utf8')); } + errors.push(`required artifact is missing: ${file}`); } catch (error) { // eslint-disable-next-line no-console console.error(`[ppl-lint-manifest] could not parse ${file}: ${error.message}`); + errors.push(`required artifact is malformed: ${file}: ${error.message}`); } return undefined; } function main() { - const target = readJson(path.join(ARTIFACTS, 'target.json')) || {}; - const detector = readJson(path.join(ARTIFACTS, 'detector-report.json')) || {}; - const backend = readJson(path.join(ARTIFACTS, 'backend-report.json')) || []; + const artifactErrors = []; + const targetRaw = readJson(path.join(ARTIFACTS, 'target.json'), artifactErrors) || {}; + const detector = + readJson(path.join(ARTIFACTS, 'detector-report.json'), artifactErrors) || {}; + const backend = + readJson(path.join(ARTIFACTS, 'backend-report.json'), artifactErrors) || []; + + let target = {}; + try { + target = normalizeTarget(targetRaw); + } catch (error) { + artifactErrors.push(`invalid target.json: ${error.message}`); + } + const executionBackend = target.executionBackend || ''; + if (target.schemaVersion !== 2 || target.legacy) { + artifactErrors.push( + `required workflow target schemaVersion must be 2, got ${JSON.stringify(target.schemaVersion)}` + ); + } + if (executionBackend !== 'standard') { + artifactErrors.push( + `required workflow target executionBackend must be "standard", got ${JSON.stringify(executionBackend)}` + ); + } + if (!target.sqlSha) { + artifactErrors.push('target sqlSha must be non-empty'); + } else if (process.env.SQL_SHA && target.sqlSha !== process.env.SQL_SHA) { + artifactErrors.push( + `target sqlSha ${JSON.stringify(target.sqlSha)} does not match workflow SQL_SHA ${JSON.stringify(process.env.SQL_SHA)}` + ); + } + if (detector.schemaVersion !== 2) { + artifactErrors.push( + `detector schemaVersion must be 2, got ${JSON.stringify(detector.schemaVersion)}` + ); + } + if (detector.executionBackend !== executionBackend) { + artifactErrors.push( + `detector executionBackend ${JSON.stringify(detector.executionBackend)} does not match target ${JSON.stringify(executionBackend)}` + ); + } + for (const field of ['engineVersion', 'grammarHash']) { + if (detector[field] !== target[field]) { + artifactErrors.push( + `detector ${field} ${JSON.stringify(detector[field])} does not match target ${JSON.stringify(target[field])}` + ); + } + } + if (!['runtime-bundle', 'compiled-simplified'].includes(detector.surface)) { + artifactErrors.push( + `detector surface must be "runtime-bundle" or "compiled-simplified", got ` + + `${JSON.stringify(detector.surface)}` + ); + } + if (!Array.isArray(detector.defaultErrorRules)) { + artifactErrors.push('detector defaultErrorRules must be an array'); + } + + let backendByKey = new Map(); + try { + backendByKey = indexBackendReport(backend, target); + } catch (error) { + artifactErrors.push(`invalid backend-report.json: ${error.message}`); + } + if (backendByKey.size === 0) { + artifactErrors.push('backend-report.json must be a non-empty array'); + } + if (!Array.isArray(detector.results) || detector.results.length === 0) { + artifactErrors.push('detector-report.json must contain a non-empty results array'); + } + const detectorKeys = new Set(); + for (const entry of Array.isArray(detector.results) ? detector.results : []) { + const key = `${entry.ruleId}::${entry.queryName}`; + if (!entry.ruleId || !entry.queryName) { + artifactErrors.push(`detector-report.json contains an invalid row ${JSON.stringify(entry)}`); + continue; + } + if (detectorKeys.has(key)) { + artifactErrors.push(`detector-report.json contains duplicate row ${key}`); + } + detectorKeys.add(key); + if (entry.executionBackend !== executionBackend) { + artifactErrors.push( + `detector row ${key} executionBackend ${JSON.stringify(entry.executionBackend)} does not match target ${JSON.stringify(executionBackend)}` + ); + } + if (!backendByKey.has(key)) { + artifactErrors.push(`detector row ${key} has no matching backend row`); + } + if (!Number.isInteger(entry.expected) || !Number.isInteger(entry.actual)) { + artifactErrors.push(`detector row ${key} must contain integer expected/actual counts`); + } else if (entry.actual !== entry.expected) { + artifactErrors.push( + `detector row ${key} count mismatch: expected ${entry.expected}, got ${entry.actual}` + ); + } + if (entry.severityMatched !== true) { + artifactErrors.push(`detector row ${key} did not match its severity assertion`); + } + if (entry.messageMatched !== true) { + artifactErrors.push(`detector row ${key} did not match its message assertion`); + } + } + for (const [key, entry] of backendByKey) { + if (!detectorKeys.has(key)) { + artifactErrors.push(`backend row ${key} has no matching detector row`); + } + const state = classifyBackendReportRow(entry); + if (state.status !== 'observed' || entry.outcome !== 'pass') { + artifactErrors.push( + `backend row ${key} did not pass its oracle (outcome=${JSON.stringify(entry.outcome)})` + ); + } + } const eventName = process.env.EVENT_NAME || ''; const osdRef = process.env.OSD_REF || 'main'; @@ -56,7 +175,8 @@ function main() { const backendResult = process.env.BACKEND_RESULT || 'unknown'; const detectorResult = process.env.DETECTOR_RESULT || 'unknown'; - const passed = backendResult === 'success' && detectorResult === 'success'; + const passed = + backendResult === 'success' && detectorResult === 'success' && artifactErrors.length === 0; // The selected validation set is the set of rules the detector run actually // evaluated (post schedule filtering). @@ -65,6 +185,7 @@ function main() { ).sort(); const manifest = { + schemaVersion: 2, mode, // A workflow_dispatch osd_ref run is pre-merge evidence, never a // branch-protection result (design §4.1.1, T11). @@ -77,11 +198,13 @@ function main() { osdSha: process.env.OSD_SHA || '', engineVersion: target.engineVersion || detector.engineVersion || '', grammarHash: target.grammarHash || detector.grammarHash || '', + executionBackend, differential: !!detector.differential, validationSet, result: { backend: backendResult, detector: detectorResult, + artifactErrors, passed, }, }; @@ -89,6 +212,10 @@ function main() { fs.writeFileSync('run-manifest.json', JSON.stringify(manifest, null, 2)); writeSummary(manifest, detector, backend); + + if (artifactErrors.length > 0) { + throw new Error(`invalid PPL lint artifacts:\n- ${artifactErrors.join('\n- ')}`); + } } /** Compact per-rule PR summary: Rule | Version | Grammar | Detector | Backend | Result. */ @@ -112,6 +239,7 @@ function writeSummary(manifest, detector, backend) { lines.push(`- SQL: \`${manifest.sqlSha || '—'}\``); lines.push(`- OSD: \`${manifest.osdSha || '—'}\` (${manifest.osdRepo} @ \`${manifest.osdRef}\`)`); lines.push(`- Backend version: \`${manifest.engineVersion || '—'}\``); + lines.push(`- Execution backend: \`${manifest.executionBackend || '—'}\``); lines.push(`- Grammar: \`${shortHash(manifest.grammarHash)}\``); lines.push( `- Result: backend **${manifest.result.backend}**, detector **${manifest.result.detector}** → ` + @@ -124,13 +252,19 @@ function writeSummary(manifest, detector, backend) { for (const r of detector.results || []) { const be = backendByKey.get(`${r.ruleId}::${r.queryName}`); const detectorCell = `${r.actual}/${r.expected}${r.severities && r.severities.length ? ` (${r.severities.join(',')})` : ''}`; - const backendCell = be - ? be.rejected - ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` - : 'accepted' - : '—'; + const backendCell = !be + ? '—' + : typeof be.rejected !== 'boolean' + ? be.outcome || 'no verdict' + : be.rejected + ? `HTTP ${be.observed ? be.observed.httpStatus : '4xx'}` + : 'accepted'; const ok = - r.actual === r.expected && (!be || (r.role === 'trigger' ? be.rejected : !be.rejected)); + r.actual === r.expected && + r.severityMatched === true && + r.messageMatched === true && + !!be && + be.outcome === 'pass'; lines.push( `| \`${r.ruleId}\` | \`${r.queryName}\` | \`${manifest.engineVersion || '—'}\` | ` + `\`${shortHash(manifest.grammarHash)}\` | ${detectorCell} | ${backendCell} | ${ok ? 'Pass' : 'Fail'} |` diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs new file mode 100644 index 00000000000..5df505f07de --- /dev/null +++ b/scripts/ppl-lint/contract-schema.mjs @@ -0,0 +1,397 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +const EXECUTION_BACKENDS = new Set(['standard', 'analytics']); +const CONTRACT_SCHEMA_VERSIONS = new Set([3, 4]); +const APPLICABLE_BACKEND_KINDS = new Set(['rejection', 'result-shape', 'advisory']); + +function isObject(value) { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function describe(value) { + return typeof value === 'string' ? `"${value}"` : JSON.stringify(value); +} + +function requireObject(value, label) { + if (!isObject(value)) { + throw new TypeError(`${label} must be a JSON object.`); + } + return value; +} + +function requireNonEmptyString(value, label) { + if (typeof value !== 'string' || value.length === 0) { + throw new TypeError(`${label} must be a non-empty string.`); + } + return value; +} + +function requireNonNegativeInteger(value, label) { + if (!Number.isInteger(value) || value < 0) { + throw new TypeError(`${label} must be a non-negative integer.`); + } + return value; +} + +function assertOptionalString(value, label) { + if (value !== undefined && (typeof value !== 'string' || value.length === 0)) { + throw new TypeError(`${label} must be a non-empty string when present.`); + } +} + +function assertBackendOracle(oracle, ruleId, executionBackend) { + const label = `[${ruleId}] ${executionBackend} backend oracle`; + requireObject(oracle, label); + const kind = requireNonEmptyString(oracle.kind, `${label}.kind`); + + if (kind === 'not-applicable') { + const reason = requireNonEmptyString(oracle.reason, `${label}.reason`); + if (reason.trim().length === 0) { + throw new TypeError(`${label}.reason must not be blank.`); + } + requireNonEmptyString(oracle.owner, `${label}.owner`); + requireNonEmptyString(oracle.issue, `${label}.issue`); + return kind; + } + if (!APPLICABLE_BACKEND_KINDS.has(kind)) { + throw new Error(`[${ruleId}] unknown ${executionBackend} backend oracle.kind "${kind}".`); + } + + if (!Number.isInteger(oracle.httpStatus) || oracle.httpStatus < 100 || oracle.httpStatus > 599) { + throw new TypeError(`${label}.httpStatus must be an integer from 100 through 599.`); + } + if (kind === 'rejection') { + const body = requireObject(oracle.body, `${label}.body`); + if (!Number.isInteger(body.status)) { + throw new TypeError(`${label}.body.status must be an integer.`); + } + if (body.error !== undefined) { + const error = requireObject(body.error, `${label}.body.error`); + assertOptionalString(error.type, `${label}.body.error.type`); + assertOptionalString(error.reason, `${label}.body.error.reason`); + } + } + if (kind === 'result-shape' && oracle.expect !== undefined) { + const expect = requireObject(oracle.expect, `${label}.expect`); + if ( + expect.datarowsNonEmpty !== undefined && + typeof expect.datarowsNonEmpty !== 'boolean' + ) { + throw new TypeError(`${label}.expect.datarowsNonEmpty must be a boolean.`); + } + if (expect.datarowsCount !== undefined) { + requireNonNegativeInteger(expect.datarowsCount, `${label}.expect.datarowsCount`); + } + assertOptionalString(expect.columnAllNull, `${label}.expect.columnAllNull`); + } + return kind; +} + +export function assertExecutionBackend(value, label = 'executionBackend') { + if (!EXECUTION_BACKENDS.has(value)) { + throw new Error( + `${label} must be "standard" or "analytics", got ${describe(value)}.` + ); + } + return value; +} + +/** + * Validate target.json and return the identity consumed by report readers. + * + * Execution identity is never inferred. Every producer in the current + * workflows writes schemaVersion 2, so an unversioned target is an incomplete + * artifact rather than a compatibility mode. + */ +export function normalizeTarget(target) { + requireObject(target, 'target'); + + const hasSchemaVersion = Object.prototype.hasOwnProperty.call(target, 'schemaVersion'); + if (!hasSchemaVersion) { + throw new Error('target.schemaVersion is required; expected 2.'); + } + + if (target.schemaVersion !== 2) { + throw new Error( + `Unsupported target schemaVersion ${describe(target.schemaVersion)}; expected 2.` + ); + } + const executionBackend = assertExecutionBackend( + target.executionBackend, + 'target.executionBackend' + ); + requireNonEmptyString(target.engineVersion, 'target.engineVersion'); + if (typeof target.grammarHash !== 'string') { + throw new TypeError('target.grammarHash must be a string.'); + } + if ( + Object.prototype.hasOwnProperty.call(target, 'grammarBundle') && + typeof target.grammarBundle !== 'string' + ) { + throw new TypeError('target.grammarBundle must be a string when present.'); + } + if ( + Object.prototype.hasOwnProperty.call(target, 'sqlSha') && + typeof target.sqlSha !== 'string' + ) { + throw new TypeError('target.sqlSha must be a string when present.'); + } + if (!Number.isInteger(target.shardCount) || target.shardCount < 1) { + throw new Error('target.shardCount must be a positive integer.'); + } + if (executionBackend === 'analytics') { + if (target.storage !== 'composite-parquet') { + throw new Error( + `analytics target.storage must be "composite-parquet", got ${describe(target.storage)}.` + ); + } + const analyticsStack = requireObject( + target.analyticsStack, + 'analytics target.analyticsStack' + ); + requireNonEmptyString( + analyticsStack.source, + 'analytics target.analyticsStack.source' + ); + const attestation = requireObject( + target.routeAttestation, + 'analytics target.routeAttestation' + ); + for (const check of [ + 'pluginsVerified', + 'clusterSettingsVerified', + 'fixtureIndicesVerified', + 'explainVerified', + 'profiledExecutionVerified', + ]) { + if (attestation[check] !== true) { + throw new Error(`analytics target.routeAttestation.${check} must be true.`); + } + } + } else if (target.storage !== 'lucene') { + throw new Error( + `standard target.storage must be "lucene", got ${describe(target.storage)}.` + ); + } + + return { + schemaVersion: 2, + executionBackend, + engineVersion: target.engineVersion, + grammarHash: target.grammarHash, + grammarBundle: target.grammarBundle || '', + sqlSha: target.sqlSha || '', + storage: target.storage || (executionBackend === 'standard' ? 'lucene' : ''), + shardCount: target.shardCount, + analyticsStack: target.analyticsStack, + routeAttestation: target.routeAttestation, + legacy: false, + }; +} + +export function assertContractSchema(spec) { + requireObject(spec, 'contract'); + if (!CONTRACT_SCHEMA_VERSIONS.has(spec.schemaVersion)) { + throw new Error( + `Unsupported contract schemaVersion ${describe(spec.schemaVersion)}; expected 3 or 4.` + ); + } + requireNonEmptyString(spec.ruleId, 'contract.ruleId'); + return spec.schemaVersion; +} + +/** + * Require a selected expectation to cover every top-level query exactly once. + * JSON object keys are unique after parsing, so set equality establishes the + * one-to-one query identity needed by both backend and detector readers. + */ +export function assertExactQueryCoverage(spec, expectation) { + assertContractSchema(spec); + requireObject(spec.queries, `[${spec.ruleId}] contract.queries`); + requireObject(expectation, `[${spec.ruleId}] selected expectation`); + requireObject(expectation.queries, `[${spec.ruleId}] selected expectation.queries`); + + const contractKeys = Object.keys(spec.queries).sort(); + const expectationKeys = Object.keys(expectation.queries).sort(); + if (contractKeys.length === 0) { + throw new Error(`[${spec.ruleId}] contract.queries must not be empty.`); + } + const contractSet = new Set(contractKeys); + const expectationSet = new Set(expectationKeys); + const missing = contractKeys.filter((key) => !expectationSet.has(key)); + const extra = expectationKeys.filter((key) => !contractSet.has(key)); + + if (missing.length > 0 || extra.length > 0) { + const details = []; + if (missing.length > 0) { + details.push(`missing from expectation: ${missing.join(', ')}`); + } + if (extra.length > 0) { + details.push(`not present in contract.queries: ${extra.join(', ')}`); + } + throw new Error(`[${spec.ruleId}] query coverage must be exact (${details.join('; ')}).`); + } + return contractKeys; +} + +/** + * Resolve only the route-specific backend oracle. Detector count, severity, and + * message assertions remain on the shared query expectation and are returned + * unchanged for either execution backend. + */ +export function resolveBackendOracle(spec, queryExpectation, executionBackend) { + const schemaVersion = assertContractSchema(spec); + assertExecutionBackend(executionBackend); + requireObject(queryExpectation, `[${spec.ruleId}] query expectation`); + + requireNonNegativeInteger( + queryExpectation.detectorCount, + `[${spec.ruleId}] detectorCount` + ); + if ( + Object.prototype.hasOwnProperty.call(queryExpectation, 'severity') && + (typeof queryExpectation.severity !== 'string' || + queryExpectation.severity.length === 0) + ) { + throw new TypeError( + `[${spec.ruleId}] severity must be a non-empty string when present.` + ); + } + if ( + Object.prototype.hasOwnProperty.call(queryExpectation, 'matchMessage') && + typeof queryExpectation.matchMessage !== 'string' + ) { + throw new TypeError(`[${spec.ruleId}] matchMessage must be a string when present.`); + } + + const detector = { + count: queryExpectation.detectorCount, + severity: queryExpectation.severity, + matchMessage: queryExpectation.matchMessage, + }; + + let oracle; + let missingReason; + if (schemaVersion === 3) { + if (executionBackend === 'standard') { + oracle = queryExpectation.backend; + missingReason = 'schema-v3 query has no backend oracle'; + } else { + missingReason = + 'schema-v3 backend oracles are standard-only; no analytics oracle is defined'; + } + } else { + if ( + Object.prototype.hasOwnProperty.call(queryExpectation, 'backends') && + !isObject(queryExpectation.backends) + ) { + throw new TypeError(`[${spec.ruleId}] backends must be a JSON object.`); + } + for (const backend of Object.keys(queryExpectation.backends || {})) { + assertExecutionBackend(backend, `[${spec.ruleId}] backends key`); + } + oracle = queryExpectation.backends && queryExpectation.backends[executionBackend]; + missingReason = `schema-v4 query has no ${executionBackend} backend oracle`; + } + + if (oracle === undefined) { + return { + status: 'coverage-missing', + executionBackend, + detector, + oracle: undefined, + reason: missingReason, + }; + } + + const kind = assertBackendOracle(oracle, spec.ruleId, executionBackend); + if (kind === 'not-applicable') { + return { + status: 'not-applicable', + executionBackend, + detector, + oracle, + reason: oracle.reason, + }; + } + + return { + status: 'applicable', + executionBackend, + detector, + oracle, + reason: undefined, + }; +} + +export function backendReportKey(entry) { + requireObject(entry, 'backend report row'); + const ruleId = requireNonEmptyString(entry.ruleId, 'backend report row.ruleId'); + const queryName = requireNonEmptyString(entry.queryName, 'backend report row.queryName'); + return `${ruleId}::${queryName}`; +} + +/** + * Read the backend observation state without coercing a missing verdict to + * acceptance. Explicit infrastructure/coverage states take precedence even if + * a malformed row also happens to contain `rejected`. + */ +export function classifyBackendReportRow(entry) { + requireObject(entry, 'backend report row'); + if (entry.outcome === 'not-applicable' || entry.kind === 'not-applicable') { + return { status: 'not-applicable', rejected: undefined }; + } + if (entry.outcome === 'coverage-missing' || entry.kind === 'coverage-missing') { + return { status: 'coverage-missing', rejected: undefined }; + } + if (entry.outcome === 'error' || typeof entry.rejected !== 'boolean') { + return { status: 'error', rejected: undefined }; + } + return { status: 'observed', rejected: entry.rejected }; +} + +/** + * Validate and index the historical bare-array backend report. + * + * Every row carries the same explicit backend identity as its schema-v2 target. + */ +export function indexBackendReport(entries, targetIdentity) { + if (!Array.isArray(entries)) { + throw new TypeError('backend report must be a JSON array.'); + } + requireObject(targetIdentity, 'normalized target identity'); + assertExecutionBackend( + targetIdentity.executionBackend, + 'normalized target identity.executionBackend' + ); + + const byKey = new Map(); + for (let index = 0; index < entries.length; index += 1) { + const entry = entries[index]; + const key = backendReportKey(entry); + const hasIdentity = Object.prototype.hasOwnProperty.call(entry, 'executionBackend'); + if (!hasIdentity) { + throw new Error( + `backend report row ${key} is missing executionBackend for a schema-v2 target.` + ); + } + const rowBackend = assertExecutionBackend( + entry.executionBackend, + `backend report row ${key}.executionBackend` + ); + if (rowBackend !== targetIdentity.executionBackend) { + throw new Error( + `backend report row ${key} executionBackend "${rowBackend}" does not match ` + + `target "${targetIdentity.executionBackend}".` + ); + } + if (byKey.has(key)) { + throw new Error(`duplicate backend report key "${key}".`); + } + byKey.set(key, entry); + } + return byKey; +} diff --git a/scripts/ppl-lint/drift.mjs b/scripts/ppl-lint/drift.mjs index 3fe9c6fa5dd..560e683975c 100644 --- a/scripts/ppl-lint/drift.mjs +++ b/scripts/ppl-lint/drift.mjs @@ -31,22 +31,28 @@ /** Every drift class this module can emit, with a stable one-line meaning. */ export const DRIFT_CLASSES = { GRAMMAR_RULE_MISSING: 'grammar-rule-missing', + EXECUTION_BACKEND_DIVERGENCE: 'execution-backend-divergence', + BACKEND_ORACLE_MISMATCH: 'backend-oracle-mismatch', ENGINE_RELAXED: 'engine-relaxed', ENGINE_PARTIALLY_RELAXED: 'engine-partially-relaxed', ENGINE_TIGHTENED: 'engine-tightened', ENGINE_MESSAGE_CHANGED: 'engine-message-changed', DETECTOR_SILENT: 'detector-silent', DETECTOR_NOISY: 'detector-noisy', + DETECTOR_COUNT_MISMATCH: 'detector-count-mismatch', + DETECTOR_MESSAGE_MISMATCH: 'detector-message-mismatch', VERSION_SCOPE_TOO_NARROW: 'version-scope-too-narrow', SEVERITY_MISMATCH: 'severity-mismatch', }; /** Remediation actions, phrased as what the linter engineer changes. */ export const REMEDIATIONS = { + ALIGN_EXECUTION_BACKENDS: 'align-execution-backends', DISABLE_RULE: 'disable-rule', VERSION_SCOPE_RULE: 'version-scope-rule', UPDATE_DETECTOR: 'update-detector', UPDATE_CONTRACT: 'update-contract', + REVIEW_BACKEND_ORACLE: 'review-backend-oracle', }; /** OSD paths an engineer edits, kept in one place so a move is a one-line fix. */ @@ -186,6 +192,119 @@ function describeObservation(observed) { return `detector ${detector}, engine ${backend}`; } +function describeBackendVerdict(observed) { + if (!observed || typeof observed.backendRejected !== 'boolean') { + return 'did not produce a verdict'; + } + if (!observed.backendRejected) { + return 'ACCEPTED'; + } + const type = observed.backendType ? ` (${observed.backendType})` : ''; + return `REJECTED${type}`; +} + +function executionBackendRemediation(ruleId, detectorPath) { + return { + action: REMEDIATIONS.ALIGN_EXECUTION_BACKENDS, + target: `analytics backend and ${detectorFile(ruleId, detectorPath)}`, + detail: + `Keep the OpenSearch version bounds unchanged. Review the route-specific backend oracles, ` + + `then either align analytics behavior with standard, narrow the detector to behavior common ` + + `to both routes, disable the rule for every route, or add a reliable execution-backend signal ` + + `to the lint context before emitting route-specific diagnostics.`, + }; +} + +/** + * Compare the two execution routes for one query on the same engine candidate. + * This is deliberately separate from product-version drift: a route difference + * cannot justify changing an OpenSearch version range. + */ +export function classifyExecutionBackendDivergence({ + ruleId, + version, + queryName, + role = 'trigger', + query, + standardObserved, + analyticsObserved, + standardLeg, + analyticsLeg, + grammarHash, + detectorPath, +}) { + if ( + !standardObserved || + !analyticsObserved || + typeof standardObserved.backendRejected !== 'boolean' || + typeof analyticsObserved.backendRejected !== 'boolean' || + standardObserved.backendRejected === analyticsObserved.backendRejected + ) { + return null; + } + + const where = `${ruleId} @ ${version} [${queryName}]`; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + executionBackend: 'analytics', + baselineExecutionBackend: 'standard', + executionBackends: ['standard', 'analytics'], + driftClass: DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE, + evidence: + `${where}: standard ${describeBackendVerdict(standardObserved)} while analytics ` + + `${describeBackendVerdict(analyticsObserved)} on the same engine candidate and runtime ` + + `grammar${grammarHash ? ` (${grammarHash})` : ''}` + + `${standardLeg || analyticsLeg ? `; legs ${standardLeg || 'standard'} / ${analyticsLeg || 'analytics'}` : ''}.`, + remediation: executionBackendRemediation(ruleId, detectorPath), + }; +} + +function backendOracleRemediation(executionBackend) { + return { + action: REMEDIATIONS.REVIEW_BACKEND_ORACLE, + target: `${executionBackend} backend and this contract file`, + detail: + `Review the captured raw response and determine whether the backend regressed or the reviewed ` + + `${executionBackend} oracle is stale. Restore the backend behavior when the status/result change ` + + `is unintended; update the oracle only after confirming the new behavior is intentional. Keep ` + + `the OpenSearch version bounds unchanged.`, + }; +} + +function classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected, + detectorPath, + reason, +}) { + const expected = expectedRejected ? 'REJECTION' : 'ACCEPTANCE'; + return { + ruleId, + version, + driftVersion: version, + queryName, + role, + query, + executionBackend: 'analytics', + executionBackends: ['analytics'], + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${ruleId} @ ${version} [${queryName}]: analytics ${describeBackendVerdict(observed)}, ` + + `but ${reason || `its reviewed oracle requires ${expected}`}.`, + remediation: backendOracleRemediation('analytics'), + }; +} + /** * Report a parser rule the detector walks that the candidate grammar no longer * defines. Exported so a caller can raise it ONCE per rule/version — the fact is @@ -203,6 +322,7 @@ export function classifyGrammarDrift({ role = 'trigger', query, detectorPath, + executionBackend = 'standard', }) { if (!Array.isArray(requiredParserRules) || !Array.isArray(parserRuleNames)) { return null; @@ -222,9 +342,10 @@ export function classifyGrammarDrift({ queryName, role, query, + executionBackend, driftClass: DRIFT_CLASSES.GRAMMAR_RULE_MISSING, evidence: - `${ruleId} @ ${version}${at}: the candidate grammar has no parser rule(s) ${missingList}, ` + + `${ruleId} @ ${version} (${executionBackend})${at}: the candidate grammar has no parser rule(s) ${missingList}, ` + `which this rule's detector walks.` + (observed && observed.detectorCount !== undefined ? ` ${describeObservation(observed)}.` : ''), remediation: { @@ -281,6 +402,7 @@ export function classifyRelaxationScope({ detectorFlagged = false, wiring, detectorPath, + executionBackend = 'standard', }) { if (relaxedTriggers.length === 0) { return null; @@ -292,6 +414,7 @@ export function classifyRelaxationScope({ version, driftVersion: version, role: 'trigger', + executionBackend, scope: { relaxed: [...relaxedTriggers], holding: [...holdingTriggers], @@ -306,6 +429,17 @@ export function classifyRelaxationScope({ const observed = relaxedTriggers.length + holdingTriggers.length; const basis = `${relaxedTriggers.length} of ${observed} observed trigger(s) relaxed`; + if (executionBackend === 'analytics') { + return { + ...base, + driftClass: DRIFT_CLASSES.EXECUTION_BACKEND_DIVERGENCE, + executionBackends: ['analytics'], + evidence: + `${where}: analytics accepted ${basis}; this is route-specific behavior, not product-version drift.`, + remediation: executionBackendRemediation(ruleId, detectorPath), + }; + } + // --- Partial: some triggers relaxed, others still rejected ------------------ // The engine fixed part of the condition. Scoping the rule out of this version // would ship a false negative on everything in `holding`, so the action is to @@ -413,13 +547,14 @@ export function classifyDrift(input) { expectedBackend, detectorPath, controlAlsoRejected, + executionBackend = 'standard', } = input; const detectorFlagged = (observed.detectorCount || 0) > 0; const expectFlagged = (expected.detectorCount || 0) > 0; const backendRejected = observed.backendRejected; - const where = `${ruleId} @ ${version} [${queryName}]`; - const base = { ruleId, version, queryName, role, query, driftVersion: version }; + const where = `${ruleId} @ ${version} (${executionBackend}) [${queryName}]`; + const base = { ruleId, version, queryName, role, query, driftVersion: version, executionBackend }; // --- 1. Did the grammar move out from under the detector? ------------------- // A detector that walks a parser rule the candidate grammar no longer defines @@ -435,6 +570,7 @@ export function classifyDrift(input) { role, query, detectorPath, + executionBackend, }); if (grammarDrift) { return grammarDrift; @@ -446,6 +582,7 @@ export function classifyDrift(input) { // engine rejects the trigger, the version window is too narrow and users on // this version get no diagnostic. const inScope = versionInAppliesTo(wiring && wiring.appliesTo, version); + const expectRejection = expected.backendKind === 'rejection'; if (!inScope) { // A trigger the engine rejects normally means the version window is too // narrow. But if the rule's CONTROL — a valid query using the same command — @@ -455,6 +592,19 @@ export function classifyDrift(input) { // what is really "unsupported command", so that case is correctly silent: // the version window is doing its job. if (role === 'trigger' && backendRejected === true && controlAlsoRejected !== true) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: false, + detectorPath, + reason: 'the standard product-version rule is inactive for this engine candidate', + }); + } return { ...base, driftClass: DRIFT_CLASSES.VERSION_SCOPE_TOO_NARROW, @@ -492,7 +642,7 @@ export function classifyDrift(input) { `the version context rather than deciding on its own, and remember OSD's version filter runs ` + `a rule when the cluster version is unknown — so this also fires for users whose version ` + `could not be resolved.` + - (backendRejected === true + (backendRejected === true && executionBackend === 'standard' ? ` The engine does reject this query, so widening appliesTo in ${OSD_PATHS.catalog} may be` + ` the right fix instead.` : ''), @@ -504,13 +654,24 @@ export function classifyDrift(input) { } // --- 3. Behavioral flips: the engine changed its verdict -------------------- - const expectRejection = expected.backendKind === 'rejection'; // 3a. The engine now ACCEPTS what the contract pinned as a rejection. Any // diagnostic the linter still emits is a false positive shipped to users — // the single most damaging drift, so it is reported even when the detector // count happens to match the stale expectation. if (role === 'trigger' && expectRejection && backendRejected === false) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: true, + detectorPath, + }); + } return { ...base, // A single relaxed trigger cannot tell a full fix from a partial one, and the @@ -548,6 +709,18 @@ export function classifyDrift(input) { // 3b. The engine now REJECTS what the contract pinned as valid. A control that // started failing means the linter is silently missing a real error. if (!expectRejection && backendRejected === true) { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: false, + detectorPath, + }); + } return { ...base, driftClass: DRIFT_CLASSES.ENGINE_TIGHTENED, @@ -590,6 +763,10 @@ export function classifyDrift(input) { const detectorMatches = (observed.detectorCount || 0) === (expected.detectorCount || 0); if (backendRejected === true && expectRejection && expectedBackend && detectorMatches) { const expectedError = (expectedBackend.body && expectedBackend.body.error) || {}; + const statusChanged = + expectedBackend.httpStatus !== undefined && + observed.backendStatus !== undefined && + expectedBackend.httpStatus !== observed.backendStatus; const typeChanged = expectedError.type !== undefined && observed.backendType !== undefined && @@ -598,6 +775,16 @@ export function classifyDrift(input) { expectedError.reason !== undefined && observed.backendReason !== undefined && expectedError.reason !== observed.backendReason; + if (statusChanged) { + return { + ...base, + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${where}: the backend still rejects the query, but its HTTP status changed from ` + + `${expectedBackend.httpStatus} to ${observed.backendStatus}.`, + remediation: backendOracleRemediation(executionBackend), + }; + } if (typeChanged || reasonChanged) { const parts = []; if (typeChanged) parts.push(`error.type "${expectedError.type}" -> "${observed.backendType}"`); @@ -620,6 +807,36 @@ export function classifyDrift(input) { } } + // Result-shape assertions and other detailed backend oracles can change while + // the coarse accepted/rejected verdict stays the same. The Java observer + // records that assertion failure explicitly; it must not be treated as + // agreement merely because a boolean verdict is still available. + if (observed.backendOutcome === 'observed-mismatch' || observed.backendOutcome === 'fail') { + if (executionBackend === 'analytics') { + return classifyAnalyticsOracleMismatch({ + ruleId, + version, + queryName, + role, + query, + observed, + expectedRejected: expectRejection, + detectorPath, + reason: `its reviewed backend oracle did not match: ${ + observed.backendMismatch || 'unspecified assertion mismatch' + }`, + }); + } + return { + ...base, + driftClass: DRIFT_CLASSES.BACKEND_ORACLE_MISMATCH, + evidence: + `${where}: the backend kept the same coarse verdict but failed its detailed oracle — ` + + `${observed.backendMismatch || 'unspecified assertion mismatch'}.`, + remediation: backendOracleRemediation(executionBackend), + }; + } + // --- 5. Detector-only disagreements ---------------------------------------- // The engine behaved as pinned, so any mismatch is on the linter side. if (expectFlagged && !detectorFlagged) { @@ -670,13 +887,31 @@ export function classifyDrift(input) { }; } - // --- 6. Right verdict, wrong severity -------------------------------------- + if (observed.detectorCount !== expected.detectorCount) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_COUNT_MISMATCH, + evidence: + `${where}: expected exactly ${expected.detectorCount} diagnostic(s), but the detector ` + + `emitted ${observed.detectorCount} while the backend behaved as pinned.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Restore the detector to emit exactly ${expected.detectorCount} diagnostic(s) for this ` + + `query, or re-pin detectorCount only after confirming the changed multiplicity is intentional.`, + }, + }; + } + + // --- 6. Right verdict, wrong severity/message ------------------------------ if ( expected.severity && detectorFlagged && - Array.isArray(observed.severities) && - observed.severities.length > 0 && - !observed.severities.every((s) => s === expected.severity) + (observed.severityMatched === false || + (Array.isArray(observed.severities) && + observed.severities.length > 0 && + !observed.severities.every((s) => s === expected.severity))) ) { return { ...base, @@ -694,6 +929,23 @@ export function classifyDrift(input) { }; } + if (expected.matchMessage && observed.messageMatched !== true) { + return { + ...base, + driftClass: DRIFT_CLASSES.DETECTOR_MESSAGE_MISMATCH, + evidence: + `${where}: the detector diagnostic no longer contains the contracted message fragment ` + + `${JSON.stringify(expected.matchMessage)}.`, + remediation: { + action: REMEDIATIONS.UPDATE_DETECTOR, + target: detectorFile(ruleId, detectorPath), + detail: + `Restore the diagnostic message asserted by this contract, or update matchMessage only ` + + `after reviewing the new user-facing wording.`, + }, + }; + } + return null; } @@ -720,9 +972,11 @@ export function formatDriftReport(drifts) { // Most urgent action first: a false positive already reaching users outranks a // stale pinned string. const order = [ + REMEDIATIONS.ALIGN_EXECUTION_BACKENDS, REMEDIATIONS.DISABLE_RULE, REMEDIATIONS.VERSION_SCOPE_RULE, REMEDIATIONS.UPDATE_DETECTOR, + REMEDIATIONS.REVIEW_BACKEND_ORACLE, REMEDIATIONS.UPDATE_CONTRACT, ]; for (const action of order) { @@ -730,7 +984,11 @@ export function formatDriftReport(drifts) { if (!group || group.length === 0) continue; lines.push(`## ${action} (${group.length})`); for (const drift of group) { - lines.push(`- [${drift.driftClass}] ${drift.evidence}`); + const backend = + Array.isArray(drift.executionBackends) && drift.executionBackends.length > 1 + ? drift.executionBackends.join(' vs ') + : drift.executionBackend || 'standard'; + lines.push(`- [${drift.driftClass}] [${backend}] ${drift.evidence}`); lines.push(` FIX (${drift.remediation.target}): ${drift.remediation.detail}`); // A rule-level finding (e.g. a grammar rename) has no single query behind it. if (drift.query) { diff --git a/scripts/ppl-lint/probe-discovery-backend.mjs b/scripts/ppl-lint/probe-discovery-backend.mjs index 8ebb0d25d75..dc5d5282a8a 100644 --- a/scripts/ppl-lint/probe-discovery-backend.mjs +++ b/scripts/ppl-lint/probe-discovery-backend.mjs @@ -15,9 +15,9 @@ * which is what this does, in the same report shape the aggregator and the labeler * already read. * - * Emits `[{ ruleId, queryName, rejected, outcome, observed: { httpStatus, type, - * reason } }]`, matching `backend-report.json` so `label-discovery.mjs` can read - * either source without a special case. + * Emits `[{ ruleId, queryName, executionBackend, rejected, outcome, observed: + * { httpStatus, type, reason } }]`, matching `backend-report.json` so + * `label-discovery.mjs` can read either source without a special case. * * The `outcome` field carries the distinction everything downstream depends on: * @@ -172,6 +172,7 @@ async function main() { queryName: entry.name || `discovery-${i}`, role: 'discovery', query: entry.query, + executionBackend: 'standard', ...verdict, }; }); diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index dcdfe5e4a1d..3606ee6510f 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -25,7 +25,7 @@ * regardless of where the entry script lives. That is what lets this SQL-owned * `.mjs` load OSD's Node-safe headless lint API without OSD's own Jest. * - * This is the detector half of a schema-v3 cross-repository differential + * This is the detector half of a schema-v3/v4 cross-repository differential * contract (see integ-test/src/test/resources/ppl-lint/contracts/*.spec.json). * Unlike the earlier PoC — which linted with the compiled analyzer or a * hand-rolled reparse against OSD `main`'s checked-in grammar — it lints against @@ -78,6 +78,15 @@ import fs from 'fs'; import path from 'path'; import { createRequire } from 'module'; +import { + assertContractSchema, + assertExactQueryCoverage, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, + resolveBackendOracle, +} from './contract-schema.mjs'; + // OSD's Node-safe headless lint API (design §4.3). Deep-path module; resolved // against the OSD checkout root, not this script's SQL-repo location. const HEADLESS_MODULE = 'src/plugins/data/public/antlr/opensearch_ppl/headless_ppl_lint'; @@ -125,6 +134,37 @@ function fatal(message) { process.exit(2); } +function loadContractFile(file) { + try { + const spec = JSON.parse(fs.readFileSync(file, 'utf8')); + assertContractSchema(spec); + const grammarSurface = spec.grammarSurface || 'runtime-bundle'; + if (!['runtime-bundle', 'compiled-simplified', 'both'].includes(grammarSurface)) { + throw new Error( + `[${spec.ruleId}] grammarSurface must be "runtime-bundle", ` + + `"compiled-simplified", or "both", got ${JSON.stringify(grammarSurface)}.` + ); + } + if (!Array.isArray(spec.expectations) || spec.expectations.length === 0) { + throw new TypeError(`[${spec.ruleId}] expectations must be a non-empty array.`); + } + for (const expectation of spec.expectations) { + assertExactQueryCoverage(spec, expectation); + for (const queryExpectation of Object.values(expectation.queries)) { + // Validate every declared oracle, including ranges not selected by this + // target. Missing route coverage is a supported state; malformed route + // names and oracle kinds are not. + resolveBackendOracle(spec, queryExpectation, 'standard'); + resolveBackendOracle(spec, queryExpectation, 'analytics'); + } + } + return { file, spec }; + } catch (error) { + fatal(`Invalid contract ${file}: ${error.message}`); + } + return undefined; // unreachable +} + /** Load every *.spec.json under the contract dir, honoring manifest.json if present. */ function loadContracts() { const dir = process.env.PPL_LINT_CONTRACT_DIR; @@ -134,7 +174,7 @@ function loadContracts() { if (!fs.existsSync(single)) { fatal(`Contract file not found: ${single}`); } - return [{ file: single, spec: JSON.parse(fs.readFileSync(single, 'utf8')) }]; + return [loadContractFile(single)]; } if (!dir) { @@ -147,7 +187,12 @@ function loadContracts() { const manifestPath = path.join(dir, 'manifest.json'); let files; if (fs.existsSync(manifestPath)) { - const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + let manifest; + try { + manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8')); + } catch (error) { + fatal(`Invalid contract manifest ${manifestPath}: ${error.message}`); + } if (!Array.isArray(manifest.contracts)) { fatal(`manifest.json must have a "contracts" array of file names.`); } @@ -164,7 +209,7 @@ function loadContracts() { if (!fs.existsSync(file)) { fatal(`Contract referenced by manifest not found: ${file}`); } - return { file, spec: JSON.parse(fs.readFileSync(file, 'utf8')) }; + return loadContractFile(file); }); } @@ -253,7 +298,7 @@ function loadOsd() { } /** Load the candidate grammar bundle + deserialize it once (fail loud; CI has no fallback). */ -function loadCandidateGrammar(osd) { +function loadCandidateGrammar(osd, target) { const bundlePath = process.env.PPL_LINT_GRAMMAR_BUNDLE; if (!bundlePath) { fatal( @@ -270,6 +315,18 @@ function loadCandidateGrammar(osd) { } catch (error) { fatal(`Could not parse grammar bundle ${bundlePath}: ${error.message}`); } + if (bundle.grammarHash !== target.grammarHash) { + fatal( + `Candidate grammar hash ${JSON.stringify(bundle.grammarHash)} does not match target ` + + `${JSON.stringify(target.grammarHash)}.` + ); + } + if (target.grammarBundle && path.basename(bundlePath) !== target.grammarBundle) { + fatal( + `Candidate grammar filename "${path.basename(bundlePath)}" does not match target ` + + `"${target.grammarBundle}".` + ); + } try { return osd.deserializeBundleOrThrow(bundle); } catch (error) { @@ -278,38 +335,44 @@ function loadCandidateGrammar(osd) { return undefined; // unreachable } -/** Read the target manifest (engineVersion + grammarHash) written beside the bundle. */ +/** Read and validate the target identity written beside the grammar bundle. */ function loadTarget() { const targetPath = process.env.PPL_LINT_TARGET_MANIFEST; - if (targetPath && fs.existsSync(targetPath)) { - try { - return JSON.parse(fs.readFileSync(targetPath, 'utf8')); - } catch (error) { - log(`WARN: could not parse target manifest ${targetPath}: ${error.message}`); - } + if (!targetPath) { + fatal('PPL_LINT_TARGET_MANIFEST is required.'); + } + if (!fs.existsSync(targetPath)) { + fatal(`Target manifest not found: ${targetPath}`); + } + try { + return normalizeTarget(JSON.parse(fs.readFileSync(targetPath, 'utf8'))); + } catch (error) { + fatal(`Invalid target manifest ${targetPath}: ${error.message}`); } - // Back-compat / local runs without a target manifest. - return { engineVersion: process.env.PPL_SQL_VERSION || '', grammarHash: '' }; + return undefined; // unreachable } /** Index the backend report by `${ruleId}::${queryName}` for the differential. */ -function loadBackendReport() { +function loadBackendReport(target) { const reportPath = process.env.PPL_LINT_BACKEND_REPORT; - if (!reportPath || !fs.existsSync(reportPath)) { + if (!reportPath) { return undefined; } + if (!fs.existsSync(reportPath)) { + fatal(`Backend report not found: ${reportPath}`); + } let entries; try { entries = JSON.parse(fs.readFileSync(reportPath, 'utf8')); } catch (error) { - log(`WARN: could not parse backend report ${reportPath}: ${error.message}`); - return undefined; + fatal(`Could not parse backend report ${reportPath}: ${error.message}`); } - const byKey = new Map(); - for (const entry of Array.isArray(entries) ? entries : []) { - byKey.set(`${entry.ruleId}::${entry.queryName}`, entry); + try { + return indexBackendReport(entries, target); + } catch (error) { + fatal(`Invalid backend report ${reportPath}: ${error.message}`); } - return byKey; + return undefined; // unreachable } /** Coerce "3.8.0-SNAPSHOT" / "3.8" to a comparable [major, minor, patch]. */ @@ -372,7 +435,7 @@ function versionMatchesRange(range, version) { * Exactly one must match (design §5.3): zero means the rule test does not cover * this version; more than one means overlapping ranges. Both fail. */ -function selectExpectation(spec, version, isCalcite, failures) { +function selectExpectation(spec, version, isCalcite, failures, { allowMissing = false } = {}) { const expectations = spec.expectations || []; const matches = expectations.filter((exp) => { if (!versionMatchesRange(exp.version, version)) return false; @@ -384,10 +447,13 @@ function selectExpectation(spec, version, isCalcite, failures) { } const label = version || 'unknown'; if (matches.length === 0) { - failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); + if (!allowMissing) { + failures.push(`[${spec.ruleId}] no version expectation matches backend version ${label}.`); + } } else { - failures.push( - `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} (exactly one required).` + fatal( + `[${spec.ruleId}] ${matches.length} expectations match backend version ${label} ` + + '(exactly one required).' ); } return undefined; @@ -490,6 +556,9 @@ function buildContext(spec, engineVersion) { function main() { const schedule = process.env.PPL_LINT_SCHEDULE || 'pr'; const reportPath = process.env.PPL_LINT_REPORT; + const target = loadTarget(); + const backendReport = loadBackendReport(target); + const contracts = loadContracts(); const osd = loadOsd(); const { getBundledCatalog, getDetector, lintQuery, osdRoot, surface } = osd; @@ -498,17 +567,24 @@ function main() { // The compiled surface lints with OSD's own checked-in grammar, so there is no // candidate bundle to load. On the runtime surface a missing bundle stays a hard // failure — never a quiet downgrade to the compiled grammar. - const grammar = surface === 'compiled-simplified' ? undefined : loadCandidateGrammar(osd); - const target = loadTarget(); - const engineVersion = target.engineVersion || process.env.PPL_SQL_VERSION || ''; - const backendReport = loadBackendReport(); + const grammar = + surface === 'compiled-simplified' ? undefined : loadCandidateGrammar(osd, target); + const engineVersion = target.engineVersion; + const executionBackend = target.executionBackend; + const observeAnalytics = process.env.PPL_LINT_OBSERVE_ANALYTICS === '1'; + const observeOnly = + process.env.PPL_LINT_OBSERVE_ONLY === '1' || observeAnalytics; + if (observeAnalytics && executionBackend !== 'analytics') { + fatal('PPL_LINT_OBSERVE_ANALYTICS=1 requires an analytics target.'); + } - const contracts = loadContracts(); const failures = []; // Contracts this surface did not score, recorded so the report says a rule was // skipped for surface rather than leaving its absence unexplained. const skippedForSurface = []; const report = { + schemaVersion: 2, + executionBackend, osdRoot, schedule, engineVersion, @@ -521,6 +597,8 @@ function main() { // see WHY a rule has no scored cases here. skippedForSurface, grammarHash: target.grammarHash || '', + observeAnalytics, + observeOnly, differential: !!backendReport, // Census of the rules that ship enabled at ERROR severity, read from the OSD // catalog this run linted with. The multi-version aggregator enforces its @@ -537,7 +615,7 @@ function main() { log(`OSD root: ${osdRoot}`); log( - `schedule=${schedule} engineVersion=${engineVersion || '(unset)'} ` + + `schedule=${schedule} engineVersion=${engineVersion} executionBackend=${executionBackend} ` + `grammarHash=${target.grammarHash || '(unset)'} differential=${!!backendReport} ` + `contracts=${contracts.length}` ); @@ -576,6 +654,8 @@ function main() { role: queryDef.role || 'trigger', query: (queryDef.query || '').split('{{index}}').join(index), surface, + executionBackend, + outcome: 'not-applicable', notApplicable: `contract declares grammarSurface "${contractSurface}"`, }); } @@ -588,23 +668,71 @@ function main() { } const context = buildContext(spec, engineVersion); - const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures); + const expectation = selectExpectation(spec, engineVersion, context.isCalcite, failures, { + allowMissing: observeOnly, + }); if (!expectation) { + if (!observeOnly) { + continue; + } + for (const [queryName, queryDef] of Object.entries(spec.queries || {})) { + const role = queryDef.role || 'trigger'; + const query = queryDef.query.split('{{index}}').join(index); + if (surface === 'compiled-simplified' && entry.runtimeOnly) { + report.results.push({ + ruleId, + queryName, + role, + query, + surface, + executionBackend, + outcome: 'not-applicable', + notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', + }); + continue; + } + const result = lintQuery(query, grammar, context); + const matches = (result.diagnostics || []).filter((d) => d.ruleId === ruleId); + report.results.push({ + ruleId, + queryName, + role, + query, + surface, + executionBackend, + expected: 0, + actual: matches.length, + severities: matches.map((m) => m.severity), + severityMatched: true, + messageMatched: true, + backendOracleStatus: 'coverage-missing', + expectationStatus: 'coverage-missing', + }); + } continue; } const queries = spec.queries || {}; const expectedQueries = expectation.queries || {}; - for (const queryName of Object.keys(expectedQueries)) { + try { + assertExactQueryCoverage(spec, expectation); + } catch (error) { + fatal(`Invalid contract ${file}: ${error.message}`); + } + for (const queryName of Object.keys(queries)) { const queryDef = queries[queryName]; - if (!queryDef) { - failures.push(`[${ruleId}] expectation references unknown query "${queryName}".`); - continue; - } const role = queryDef.role || 'trigger'; const query = queryDef.query.split('{{index}}').join(index); const expected = expectedQueries[queryName]; - const expectedCount = expected.detectorCount; + let oracleSelection; + try { + oracleSelection = resolveBackendOracle(spec, expected, executionBackend); + } catch (error) { + fatal(`Invalid contract ${file} query "${queryName}": ${error.message}`); + } + const expectedCount = oracleSelection.detector.count; + const expectedSeverity = oracleSelection.detector.severity; + const expectedMessage = oracleSelection.detector.matchMessage; // A `runtimeOnly` rule walks grammar productions that exist only in the // runtime bundle, so `lint_runner` skips it on the compiled surface. Its @@ -623,6 +751,8 @@ function main() { role, query, surface, + executionBackend, + outcome: 'not-applicable', notApplicable: 'runtimeOnly rule does not run on the compiled-simplified surface', }); continue; @@ -639,9 +769,12 @@ function main() { ); const severityOk = - !expected.severity || actual === 0 || matches.every((m) => m.severity === expected.severity); + !expectedSeverity || + actual === 0 || + matches.every((m) => m.severity === expectedSeverity); const messageOk = - !expected.matchMessage || matches.some((m) => (m.message || '').includes(expected.matchMessage)); + !expectedMessage || + matches.some((m) => (m.message || '').includes(expectedMessage)); const resultEntry = { ruleId, @@ -651,6 +784,10 @@ function main() { expected: expectedCount, actual, severities: matches.map((m) => m.severity), + severityMatched: severityOk, + messageMatched: messageOk, + executionBackend, + backendOracleStatus: oracleSelection.status, }; if (!ok) { @@ -659,10 +796,30 @@ function main() { ); } if (!severityOk) { - failures.push(`[${ruleId}/${queryName}] expected severity "${expected.severity}" for: ${query}`); + failures.push( + `[${ruleId}/${queryName}] expected severity "${expectedSeverity}" for: ${query}` + ); } if (!messageOk) { - failures.push(`[${ruleId}/${queryName}] expected message to contain "${expected.matchMessage}" for: ${query}`); + failures.push( + `[${ruleId}/${queryName}] expected message to contain "${expectedMessage}" for: ${query}` + ); + } + + if (oracleSelection.status === 'not-applicable') { + resultEntry.outcome = 'not-applicable'; + resultEntry.reason = oracleSelection.reason; + resultEntry.notApplicable = oracleSelection.reason; + } else if (oracleSelection.status === 'coverage-missing') { + resultEntry.outcome = 'coverage-missing'; + resultEntry.coverage = 'missing'; + resultEntry.reason = oracleSelection.reason; + resultEntry.coverageMissing = oracleSelection.reason; + if (!observeAnalytics) { + failures.push( + `[${ruleId}/${queryName}] ${executionBackend} backend coverage missing: ${oracleSelection.reason}.` + ); + } } // Differential: the observed backend behavior must agree with the observed @@ -672,56 +829,71 @@ function main() { // passes. This catches drift the two halves would otherwise hide by both // pinning to the same JSON. if (backendReport) { - const backendKind = expected.backend && expected.backend.kind; - const expectRejected = backendKind === 'rejection'; const be = backendReport.get(`${ruleId}::${queryName}`); if (!be) { failures.push(`[${ruleId}/${queryName}] no backend report entry (backend did not run this query).`); } else { - resultEntry.backendRejected = !!be.rejected; - if (!!be.rejected !== expectRejected) { - failures.push( - `[${ruleId}/${queryName}] differential: backend ${be.rejected ? 'rejected' : 'accepted'} ` + - `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` - ); - } - // Trigger cross-check: a trigger the detector flags must be one the engine - // ALSO objects to — but only where the contract claims the engine objects - // at all. - // - // For a `rejection` rule the two coincide: detector flags <-> engine - // rejects, and a disagreement means one side drifted. That is the original - // check and it is unchanged. - // - // An ADVISORY rule is different by design. It flags a query the engine - // runs happily: `head-without-sort` marks non-determinism, - // `division-by-zero` marks a silent null, `dedup-consecutive` succeeds via - // the Calcite-to-v2 fallback. "Detector flagged, backend accepted" is that - // rule working, not drift — so pairing the detector against `be.rejected` - // failed every advisory trigger unconditionally. That, not runtime cost, - // is the structural reason those contracts could only run nightly. - // - // The contracts already carry the distinction in `backend.kind`, so this - // reads data that exists rather than adding a flag. Advisory triggers keep - // full coverage from the other two assertions: the backend-kind check above - // fires if the engine starts REJECTING a query pinned as accepted, and the - // `detectorCount` assertion fires if the detector stops flagging it. Only - // the pairing rule is scoped to the rules it makes sense for. - const detectorFlagged = actual > 0; - if (role === 'trigger' && expectRejected && detectorFlagged !== !!be.rejected) { - failures.push( - `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + - `but backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` - ); - } - // A control must pass on both sides regardless of kind: it is a valid - // query the rule has to stay quiet on. Unlike a trigger, that claim does - // not vary with `backend.kind`. - if (role === 'control' && (detectorFlagged || be.rejected)) { + const backendObservation = classifyBackendReportRow(be); + if (oracleSelection.status !== 'applicable') { + // A missing or non-applicable oracle is never an acceptance claim. Keep + // any backend observation visible, but do not coerce a missing verdict + // through `!!be.rejected` or score a differential against another route. + resultEntry.backendOutcome = backendObservation.status; + } else if (backendObservation.status !== 'observed') { + resultEntry.backendOutcome = backendObservation.status; failures.push( - `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + - `and backend ${be.rejected ? 'rejected' : 'accepted'} for: ${query}` + `[${ruleId}/${queryName}] backend report has no accepted/rejected verdict ` + + `(outcome=${JSON.stringify(backendObservation.status)}).` ); + } else { + const backendKind = oracleSelection.oracle.kind; + const expectRejected = backendKind === 'rejection'; + const backendRejected = backendObservation.rejected; + resultEntry.backendRejected = backendRejected; + if (backendRejected !== expectRejected) { + failures.push( + `[${ruleId}/${queryName}] differential: backend ${backendRejected ? 'rejected' : 'accepted'} ` + + `but the contract's backend.kind="${backendKind}" expects ${expectRejected ? 'rejection' : 'acceptance'} for: ${query}` + ); + } + // Trigger cross-check: a trigger the detector flags must be one the engine + // ALSO objects to — but only where the contract claims the engine objects + // at all. + // + // For a `rejection` rule the two coincide: detector flags <-> engine + // rejects, and a disagreement means one side drifted. That is the original + // check and it is unchanged. + // + // An ADVISORY rule is different by design. It flags a query the engine + // runs happily: `head-without-sort` marks non-determinism, + // `division-by-zero` marks a silent null, `dedup-consecutive` succeeds via + // the Calcite-to-v2 fallback. "Detector flagged, backend accepted" is that + // rule working, not drift — so pairing the detector against `be.rejected` + // failed every advisory trigger unconditionally. That, not runtime cost, + // is the structural reason those contracts could only run nightly. + // + // The contracts already carry the distinction in `backend.kind`, so this + // reads data that exists rather than adding a flag. Advisory triggers keep + // full coverage from the other two assertions: the backend-kind check above + // fires if the engine starts REJECTING a query pinned as accepted, and the + // `detectorCount` assertion fires if the detector stops flagging it. Only + // the pairing rule is scoped to the rules it makes sense for. + const detectorFlagged = actual > 0; + if (role === 'trigger' && expectRejected && detectorFlagged !== backendRejected) { + failures.push( + `[${ruleId}/${queryName}] differential: trigger detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `but backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } + // A control must pass on both sides regardless of kind: it is a valid + // query the rule has to stay quiet on. Unlike a trigger, that claim does + // not vary with `backend.kind`. + if (role === 'control' && (detectorFlagged || backendRejected)) { + failures.push( + `[${ruleId}/${queryName}] differential: control must pass on both sides but detector ${detectorFlagged ? 'flagged' : 'passed'} ` + + `and backend ${backendRejected ? 'rejected' : 'accepted'} for: ${query}` + ); + } } } } @@ -746,7 +918,7 @@ function main() { fs.writeFileSync(reportPath, JSON.stringify(report, null, 2)); log(`wrote report to ${reportPath}`); } catch (error) { - log(`WARN: could not write report to ${reportPath}: ${error.message}`); + fatal(`Could not write detector report ${reportPath}: ${error.message}`); } } From a20046c0ad300f0124b01bd49daaa2d11da5fbeb Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 13:00:23 -0700 Subject: [PATCH 35/39] fix(ci): follow current analytics artifact names Signed-off-by: Hanyu Wei --- integ-test/build.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index ee208050838..b5fcd4d08e0 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -342,7 +342,7 @@ task downloadTestPplFrontendZip(type: Download) { } task downloadAnalyticsBackendLuceneZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-lucene-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-lucene-${pluginVersion}.zip" dest analyticsBackendLuceneZipDest overwrite false onlyIfModified true @@ -350,7 +350,7 @@ task downloadAnalyticsBackendLuceneZip(type: Download) { } task downloadParquetDataFormatZip(type: Download) { - src "${featureBuildBase}/1-parquet-data-format-${pluginVersion}.zip" + src "${featureBuildBase}/parquet-data-format-${pluginVersion}.zip" dest parquetDataFormatZipDest overwrite false onlyIfModified true @@ -358,7 +358,7 @@ task downloadParquetDataFormatZip(type: Download) { } task downloadCompositeEngineZip(type: Download) { - src "${featureBuildBase}/1-composite-engine-${pluginVersion}.zip" + src "${featureBuildBase}/2-composite-engine-${pluginVersion}.zip" dest compositeEngineZipDest overwrite false onlyIfModified true @@ -366,7 +366,7 @@ task downloadCompositeEngineZip(type: Download) { } task downloadAnalyticsBackendDatafusionZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-datafusion-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-datafusion-${pluginVersion}.zip" dest analyticsBackendDatafusionZipDest overwrite false onlyIfModified true From 814517b455799609d1403bd27e2ce8310ea9b196 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 13:51:22 -0700 Subject: [PATCH 36/39] fix(ci): harden analytics lint observation Signed-off-by: Hanyu Wei --- .../ppl-lint-multiversion-validation.yml | 144 ++++++++- integ-test/build.gradle | 65 +++- .../remote/PplLintRuleValidationIT.java | 19 +- scripts/ppl-lint/README.md | 2 + .../__tests__/aggregate-versions.test.mjs | 23 ++ .../__tests__/contract-schema.test.mjs | 22 ++ .../validate-pr-build-targets.test.mjs | 271 +++++++++++++++++ scripts/ppl-lint/aggregate-versions.mjs | 6 +- scripts/ppl-lint/contract-schema.mjs | 6 + .../ppl-lint/validate-pr-build-targets.mjs | 285 ++++++++++++++++++ 10 files changed, 825 insertions(+), 18 deletions(-) create mode 100644 scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs create mode 100644 scripts/ppl-lint/validate-pr-build-targets.mjs diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml index b02a2a5ecf7..de047b07569 100644 --- a/.github/workflows/ppl-lint-multiversion-validation.yml +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -557,6 +557,8 @@ jobs: needs: Get-CI-Image-Tag runs-on: ubuntu-latest timeout-minutes: 30 + env: + ANALYTICS_FEATURE_BUILD_LATEST: https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch container: image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} @@ -573,12 +575,64 @@ jobs: distribution: 'temurin' java-version: 25 - - name: Run analytics contract observation against the PR build + - name: Resolve analytics feature build + id: analytics-build run: | set -euo pipefail mkdir -p leg + requested_manifest="${ANALYTICS_FEATURE_BUILD_LATEST}/manifest.yml" + resolved_manifest=$(curl --fail --silent --show-error --location \ + --retry 3 --retry-all-errors \ + --output leg/analytics-feature-manifest.yml \ + --write-out '%{url_effective}' \ + "$requested_manifest") + artifact_root="${resolved_manifest%/manifest.yml}" + plugin_base="${artifact_root}/plugins" + native_url="${artifact_root}/dist/libopensearch_native.so" + { + echo "artifact_root=$artifact_root" + echo "plugin_base=$plugin_base" + echo "native_url=$native_url" + } >> "$GITHUB_OUTPUT" + ANALYTICS_ARTIFACT_ROOT="$artifact_root" \ + ANALYTICS_PLUGIN_BASE="$plugin_base" \ + ANALYTICS_NATIVE_URL="$native_url" \ + ANALYTICS_RESOLVED_MANIFEST="$resolved_manifest" \ + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + manifest = Path("leg/analytics-feature-manifest.yml") + context = { + "schemaVersion": 1, + "stage": "feature-build-resolved", + "sqlSha": os.environ["GITHUB_SHA"], + "executionBackend": "analytics", + "storage": "composite-parquet", + "artifactRoot": os.environ["ANALYTICS_ARTIFACT_ROOT"], + "pluginBase": os.environ["ANALYTICS_PLUGIN_BASE"], + "nativeLibraryUrl": os.environ["ANALYTICS_NATIVE_URL"], + "resolvedManifestUrl": os.environ["ANALYTICS_RESOLVED_MANIFEST"], + "manifestSha256": "sha256:" + hashlib.sha256(manifest.read_bytes()).hexdigest(), + } + Path("leg/analytics-bootstrap.json").write_text( + json.dumps(context, indent=2) + "\n", encoding="utf-8" + ) + PY + + - name: Run analytics contract observation against the PR build + id: analytics-observation + env: + ANALYTICS_FEATURE_BUILD_BASE: ${{ steps.analytics-build.outputs.plugin_base }} + ANALYTICS_NATIVE_LIB_URL: ${{ steps.analytics-build.outputs.native_url }} + run: | + set -euo pipefail chown -R 1000:1000 "$(pwd)" su "$(id -un 1000)" -c "./gradlew :integ-test:analyticsEnginePplLintIT \ + -PanalyticsFeatureBuildBase=${ANALYTICS_FEATURE_BUILD_BASE} \ + -PanalyticsNativeLibUrl=${ANALYTICS_NATIVE_LIB_URL} \ -Dppl.lint.schedule=nightly \ -Dppl.lint.observe.only=true \ -Dppl.lint.sql_sha=${GITHUB_SHA} \ @@ -586,6 +640,69 @@ jobs: -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ -Dppl.lint.target=$(pwd)/leg/target.json" + - name: Record analytics bootstrap provenance + if: ${{ always() }} + env: + OBSERVATION_OUTCOME: ${{ steps.analytics-observation.outcome }} + ANALYTICS_ARTIFACT_ROOT: ${{ steps.analytics-build.outputs.artifact_root }} + ANALYTICS_PLUGIN_BASE: ${{ steps.analytics-build.outputs.plugin_base }} + ANALYTICS_NATIVE_URL: ${{ steps.analytics-build.outputs.native_url }} + run: | + mkdir -p leg + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + context_file = Path("leg/analytics-bootstrap.json") + if context_file.exists(): + context = json.loads(context_file.read_text(encoding="utf-8")) + else: + context = { + "schemaVersion": 1, + "sqlSha": os.environ["GITHUB_SHA"], + "executionBackend": "analytics", + "storage": "composite-parquet", + "artifactRoot": os.environ.get("ANALYTICS_ARTIFACT_ROOT") or None, + "pluginBase": os.environ.get("ANALYTICS_PLUGIN_BASE") or None, + "nativeLibraryUrl": os.environ.get("ANALYTICS_NATIVE_URL") or None, + } + + def describe(file): + digest = hashlib.sha256() + with file.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return { + "path": str(file), + "size": file.stat().st_size, + "sha256": "sha256:" + digest.hexdigest(), + } + + distributions = Path("integ-test/build/distributions") + native_libraries = sorted( + Path("integ-test/build/native").glob( + "*/release/libopensearch_native.so" + ) + ) + artifacts = ( + [describe(file) for file in sorted(distributions.glob("*.zip"))] + if distributions.is_dir() + else [] + ) + artifacts.extend(describe(file) for file in native_libraries) + context["stage"] = "observation-finished" + context["outcome"] = os.environ.get("OBSERVATION_OUTCOME") or "not-run" + context["effectiveJavaLibraryPaths"] = [ + str(file.parent) for file in native_libraries + ] + context["downloadedArtifacts"] = artifacts + context_file.write_text( + json.dumps(context, indent=2) + "\n", encoding="utf-8" + ) + PY + - name: Upload analytics leg artifacts if: ${{ always() }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 @@ -629,6 +746,22 @@ jobs: - name: Checkout SQL pull request uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Download all leg artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: ppl-lint-leg-* + path: legs + + - name: Validate PR-build target identity + run: | + node scripts/ppl-lint/validate-pr-build-targets.mjs \ + --standard legs/ppl-lint-leg-pr-build/target.json \ + --analytics legs/ppl-lint-leg-pr-build-analytics/target.json \ + --standard-report legs/ppl-lint-leg-pr-build/backend-report.json \ + --analytics-report legs/ppl-lint-leg-pr-build-analytics/backend-report.json \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --schedule nightly + - name: Checkout OpenSearch-Dashboards uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: @@ -674,12 +807,6 @@ jobs: done exit 1 - - name: Download all leg artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 - with: - pattern: ppl-lint-leg-* - path: legs - # One detector pass per leg, each against THAT engine's grammar bundle. The # runner is the same SQL-owned script the single-version workflow uses, so # the detector half cannot drift between the two checks. @@ -783,6 +910,7 @@ jobs: --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ --out "$GITHUB_WORKSPACE/drift-report.json" \ --summary "$GITHUB_STEP_SUMMARY" \ + --all-rules \ --observe-analytics \ "${args[@]}" @@ -797,6 +925,8 @@ jobs: legs/**/detector-report.json legs/**/detector.log legs/**/target.json + legs/**/analytics-bootstrap.json + legs/**/analytics-feature-manifest.yml # Discovery: harvest queries from OSD's own lint tests, run both halves over them, # and report detector/engine disagreements as LEADS. diff --git a/integ-test/build.gradle b/integ-test/build.gradle index b5fcd4d08e0..9452087ace6 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -300,6 +300,7 @@ def getGeoSpatialPlugin() { ext.pluginVersion = opensearch_version.tokenize('-')[0] ext.featureBuildBase = project.findProperty('analyticsFeatureBuildBase') ?: "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildArtifactRoot = featureBuildBase.replaceFirst('/plugins/?$', '') ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip" ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip" ext.arrowBaseZipDest = "${buildDir}/distributions/arrow-base-${pluginVersion}-SNAPSHOT.zip" @@ -308,6 +309,16 @@ ext.analyticsBackendLuceneZipDest = "${buildDir}/distributions/analytics-backend ext.parquetDataFormatZipDest = "${buildDir}/distributions/parquet-data-format-${pluginVersion}-SNAPSHOT.zip" ext.compositeEngineZipDest = "${buildDir}/distributions/composite-engine-${pluginVersion}-SNAPSHOT.zip" ext.analyticsBackendDatafusionZipDest = "${buildDir}/distributions/analytics-backend-datafusion-${pluginVersion}-SNAPSHOT.zip" +ext.analyticsNativeLibUrl = project.findProperty('analyticsNativeLibUrl') ?: + "${featureBuildArtifactRoot}/dist/libopensearch_native.so" +ext.analyticsNativeLibDest = "${buildDir}/native/${pluginVersion}/release/libopensearch_native.so" +ext.analyticsNativeLibDir = project.findProperty('nativeLibPath') ? + rootProject.file(project.findProperty('nativeLibPath')).canonicalFile : + file(analyticsNativeLibDest).parentFile.canonicalFile +ext.analyticsJavaLibraryPath = [ + analyticsNativeLibDir.absolutePath, + System.getProperty('java.library.path') +].findAll { it != null && !it.isEmpty() }.join(File.pathSeparator) task downloadAnalyticsEngineZip(type: Download) { src "${featureBuildBase}/1-analytics-engine-${pluginVersion}.zip" @@ -373,6 +384,50 @@ task downloadAnalyticsBackendDatafusionZip(type: Download) { onlyIf { !project.findProperty('analyticsBackendDatafusionZip') } } +task downloadAnalyticsNativeLib(type: Download) { + src analyticsNativeLibUrl + dest analyticsNativeLibDest + // The mutable observation URL can publish another build under the same + // product version. Revalidate an existing file and never expose a partial + // download to the test cluster. + overwrite true + onlyIfModified true + tempAndMove true + retries 3 + onlyIf { !project.findProperty('nativeLibPath') } + doFirst { + def osName = System.getProperty('os.name', '').toLowerCase(Locale.ROOT) + def osArch = System.getProperty('os.arch', '').toLowerCase(Locale.ROOT) + if (!osName.contains('linux') || !(osArch in ['amd64', 'x86_64'])) { + throw new GradleException( + "The default analytics native artifact is Linux/x64 only " + + "(detected ${osName}/${osArch}); pass -PnativeLibPath=.") + } + } +} + +task validateAnalyticsNativeLib { + dependsOn downloadAnalyticsNativeLib + doLast { + File nativeLib = new File(analyticsNativeLibDir, 'libopensearch_native.so') + if (!nativeLib.isFile() || !nativeLib.canRead() || nativeLib.length() == 0) { + throw new GradleException( + "Expected a readable non-empty native library at ${nativeLib}. " + + "Pass -PnativeLibPath= " + + "or -PanalyticsNativeLibUrl=.") + } + byte[] magic = new byte[4] + int bytesRead + nativeLib.withInputStream { stream -> bytesRead = stream.read(magic) } + if (bytesRead != magic.length || + (magic[0] & 0xff) != 0x7f || (magic[1] & 0xff) != 0x45 || + (magic[2] & 0xff) != 0x4c || (magic[3] & 0xff) != 0x46) { + throw new GradleException( + "Analytics native library ${nativeLib} is not an ELF shared object.") + } + } +} + def getAnalyticsEnginePlugin() { provider { (RegularFile) (() -> file(project.findProperty('analyticsEngineZip') ?: analyticsEngineZipDest)) } } @@ -479,10 +534,9 @@ testClusters { systemProperty 'io.netty.tryReflectionSetAccessible', 'true' systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' - // Native library path for DataFusion/parquet -- pass via -PnativeLibPath=/path/to/release/ - if (project.findProperty('nativeLibPath')) { - systemProperty 'java.library.path', project.findProperty('nativeLibPath') - } + // DataFusion/parquet loads libopensearch_native.so at cluster startup. Use the + // matching feature-build artifact unless a local release directory is supplied. + systemProperty 'java.library.path', analyticsJavaLibraryPath } } @@ -547,7 +601,8 @@ task analyticsEnginePplLintIT(type: RestIntegTestTask) { useCluster testClusters.analyticsEnginePplLintIT dependsOn downloadArrowBaseZip, downloadArrowFlightRpcZip, downloadAnalyticsEngineZip, downloadCompositeEngineZip, downloadParquetDataFormatZip, - downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip + downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip, + validateAnalyticsNativeLib dependsOn ':opensearch-sql-plugin:bundlePlugin' systemProperty 'tests.analytics.parquet_indices', 'true' diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 01012b7406d..f873e4c0021 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -1205,7 +1205,9 @@ private boolean pluginComponentMatches(String component, String required) { private void verifyAnalyticsClusterSettings() throws IOException { Response nodesResponse = client().performRequest(new Request("GET", "/_nodes/settings?flat_settings=true")); - JSONObject nodes = new JSONObject(getResponseBody(nodesResponse, true)).getJSONObject("nodes"); + JSONObject nodesBody = new JSONObject(getResponseBody(nodesResponse, true)); + analyticsRouteAttestation.put("nodeSettings", nodesBody); + JSONObject nodes = nodesBody.getJSONObject("nodes"); requireAttestation(nodes.length() > 0, "node settings response contained no nodes"); for (String nodeId : nodes.keySet()) { String startupDataFormat = @@ -1239,12 +1241,12 @@ private void verifyAnalyticsClusterSettings() throws IOException { .performRequest( new Request("GET", "/_cluster/settings?flat_settings=true&include_defaults=true")); JSONObject settings = new JSONObject(getResponseBody(response, true)); + analyticsRouteAttestation.put("clusterSettings", settings); requireEffectiveSetting(settings, "cluster.pluggable.dataformat", "composite"); requireEffectiveSetting(settings, "cluster.pluggable.dataformat.enabled", "true"); requireEffectiveSetting(settings, "cluster.composite.primary_data_format", "parquet"); requireEffectiveSettingContains(settings, "cluster.composite.secondary_data_formats", "lucene"); - analyticsRouteAttestation.put("clusterSettings", settings); } private void verifyAnalyticsFixtureIndices() throws IOException { @@ -1284,11 +1286,11 @@ private void verifyAnalyticsFixtureIndices() throws IOException { Response countResponse = client().performRequest(new Request("GET", "/" + indexName + "/_count")); long count = new JSONObject(getResponseBody(countResponse, true)).getLong("count"); + documentCounts.put(indexName, count); + fixtureEvidence.put("documentCount", count); requireAttestation( count > 0, "fixture " + indexName + " contains no documents; fixture ingestion did not complete"); - documentCounts.put(indexName, count); - fixtureEvidence.put("documentCount", count); } } @@ -1346,9 +1348,12 @@ private String sha256(String value) { } private void verifyAnalyticsExplainCanaries() throws IOException { + JSONObject explainPlans = new JSONObject(); + analyticsRouteAttestation.put("explainPlans", explainPlans); for (String indexEnum : requiredIndexEnums()) { String query = analyticsCanaryQuery(indexEnum); String explained = explainQueryToString(query); + explainPlans.put(indexEnum, explained); requireAttestation( explained.contains("LogicalTableScan(table=[[opensearch,"), "fixture " + indexEnum + " did not use LogicalTableScan(opensearch): " + explained); @@ -1360,8 +1365,13 @@ private void verifyAnalyticsExplainCanaries() throws IOException { private void verifyAnalyticsProfileCanaries() throws IOException { JSONArray executionTypes = new JSONArray(); + JSONObject profiles = new JSONObject(); + analyticsRouteAttestation + .put("profileExecutionTypes", executionTypes) + .put("profiles", profiles); for (String indexEnum : requiredIndexEnums()) { JSONObject response = runProfiledPplQuery(analyticsCanaryQuery(indexEnum)); + profiles.put(indexEnum, response); JSONObject profile = response.getJSONObject("profile"); JSONArray stages = profile.getJSONObject("plan").getJSONArray("stages"); requireAttestation( @@ -1377,7 +1387,6 @@ private void verifyAnalyticsProfileCanaries() throws IOException { executionTypes.put(stage.getString("execution_type")); } } - analyticsRouteAttestation.put("profileExecutionTypes", executionTypes); } private String analyticsCanaryQuery(String indexEnum) { diff --git a/scripts/ppl-lint/README.md b/scripts/ppl-lint/README.md index db60cbe23ca..79bd5d8f1dc 100644 --- a/scripts/ppl-lint/README.md +++ b/scripts/ppl-lint/README.md @@ -89,6 +89,8 @@ OSD_REF= ./scripts/ppl-lint-rule-validation.sh PPL_LINT_SCHEDULE=nightly ./scripts/ppl-lint-rule-validation.sh # Run the corpus through the full composite/Parquet + DataFusion stack. +# The published default stack is Linux/x64; other platforms need compatible +# local plugin artifacts and -PnativeLibPath. RUN_ANALYTICS=1 ./scripts/ppl-lint-rule-validation.sh # Use locally built analytics plugins (all trailing arguments pass to Gradle). diff --git a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs index 76f64e31348..b498f594745 100644 --- a/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs +++ b/scripts/ppl-lint/__tests__/aggregate-versions.test.mjs @@ -562,6 +562,29 @@ test('analytics coverage gaps cannot hide missing raw observations', () => { assert.match(report.inconclusive[0].reasons.join(' '), /no engine verdict/); }); +test('--all-rules makes incomplete non-default observations fail as infrastructure', () => { + const contracts = writeContracts(); + const manifestFile = path.join(contracts, 'manifest.json'); + const manifest = JSON.parse(fs.readFileSync(manifestFile, 'utf8')); + manifest.defaultError = []; + fs.writeFileSync(manifestFile, JSON.stringify(manifest)); + const leg = writeLeg({ + version: '3.8.0', + defaultErrorRules: [], + cases: { control: { detector: 0, rejected: false } }, + }); + + const { status, report } = run({ + contracts, + legs: [['3.8.0', leg]], + extraArgs: ['--all-rules'], + }); + + assert.equal(status, 1); + assert.equal(report.result.enforcedInconclusive, 1); + assert.equal(report.matrix[0].status, 'inconclusive'); +}); + test('paired detector reports must be identical across execution backends', () => { const grammarHash = 'sha256:shared-runtime-grammar'; const standard = writeLeg({ diff --git a/scripts/ppl-lint/__tests__/contract-schema.test.mjs b/scripts/ppl-lint/__tests__/contract-schema.test.mjs index 6dce090d45d..1f4516ca83b 100644 --- a/scripts/ppl-lint/__tests__/contract-schema.test.mjs +++ b/scripts/ppl-lint/__tests__/contract-schema.test.mjs @@ -312,6 +312,28 @@ test('backend oracle payloads fail closed when required shapes are malformed', ( }, expected: /\.body\.status must be an integer/, }, + { + oracle: { + kind: 'rejection', + httpStatus: 400, + body: { status: 500 }, + }, + expected: /\.httpStatus must equal .*\.body\.status/, + }, + { + oracle: { + kind: 'result-shape', + httpStatus: 201, + }, + expected: /\.httpStatus must be 200/, + }, + { + oracle: { + kind: 'advisory', + httpStatus: 204, + }, + expected: /\.httpStatus must be 200/, + }, { oracle: { kind: 'result-shape', diff --git a/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs b/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs new file mode 100644 index 00000000000..430daf81797 --- /dev/null +++ b/scripts/ppl-lint/__tests__/validate-pr-build-targets.test.mjs @@ -0,0 +1,271 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { after, test } from 'node:test'; +import { fileURLToPath } from 'node:url'; + +import { + validatePrBuildArtifacts, + validatePrBuildTargetPair, +} from '../validate-pr-build-targets.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const SCRIPT = path.join(HERE, '..', 'validate-pr-build-targets.mjs'); +const tmpDirs = []; + +function standardTarget(overrides = {}) { + return { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:candidate-grammar', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend: 'standard', + storage: 'lucene', + shardCount: 1, + ...overrides, + }; +} + +function analyticsTarget(overrides = {}) { + return { + schemaVersion: 2, + sqlSha: 'candidate-sql-sha', + engineVersion: '3.8.0-SNAPSHOT', + grammarHash: 'sha256:candidate-grammar', + grammarBundle: 'ppl-grammar-bundle.json', + executionBackend: 'analytics', + storage: 'composite-parquet', + shardCount: 1, + analyticsStack: { source: 'https://example.test/analytics-build' }, + routeAttestation: { + pluginsVerified: true, + clusterSettingsVerified: true, + fixtureIndicesVerified: true, + explainVerified: true, + profiledExecutionVerified: true, + }, + ...overrides, + }; +} + +function backendReport(executionBackend, queryNames = ['trigger', 'control']) { + return queryNames.map((queryName) => ({ + ruleId: 'test-rule', + queryName, + role: queryName === 'control' ? 'control' : 'trigger', + query: + queryName === 'control' + ? 'source=test-index | head 1' + : 'source=test-index | head 0', + executionBackend, + rejected: queryName !== 'control', + observed: { + httpStatus: queryName === 'control' ? 200 : 400, + rejected: queryName !== 'control', + }, + })); +} + +function writeContractCorpus() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-target-contracts-')); + tmpDirs.push(dir); + fs.writeFileSync( + path.join(dir, 'manifest.json'), + JSON.stringify({ schemaVersion: 3, contracts: ['test-rule.spec.json'] }) + ); + fs.writeFileSync( + path.join(dir, 'test-rule.spec.json'), + JSON.stringify({ + schemaVersion: 3, + ruleId: 'test-rule', + schedule: 'pr', + queries: { + trigger: { role: 'trigger', query: 'source={{index}} | head 0' }, + control: { role: 'control', query: 'source={{index}} | head 1' }, + }, + }) + ); + return dir; +} + +after(() => { + for (const dir of tmpDirs) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('matching standard and analytics PR-build targets pass', () => { + const pair = validatePrBuildTargetPair(standardTarget(), analyticsTarget()); + assert.equal(pair.standard.executionBackend, 'standard'); + assert.equal(pair.analytics.executionBackend, 'analytics'); +}); + +test('the two PR-build target roles require their exact backend identities', () => { + assert.throws( + () => validatePrBuildTargetPair(analyticsTarget(), analyticsTarget()), + /standard PR-build target executionBackend must be "standard"/ + ); + assert.throws( + () => validatePrBuildTargetPair(standardTarget(), standardTarget()), + /analytics PR-build target executionBackend must be "analytics"/ + ); +}); + +test('both PR-build targets require the same non-empty SQL SHA', () => { + assert.throws( + () => validatePrBuildTargetPair(standardTarget({ sqlSha: '' }), analyticsTarget()), + /both report a non-empty SQL SHA/ + ); + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget(), + analyticsTarget({ sqlSha: 'different-sql-sha' }) + ), + /report different values for SQL SHA/ + ); +}); + +test('both PR-build targets require the same engine version', () => { + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget(), + analyticsTarget({ engineVersion: '3.9.0-SNAPSHOT' }) + ), + /report different values for engine version/ + ); +}); + +test('both PR-build targets require the same non-empty grammar hash', () => { + assert.throws( + () => validatePrBuildTargetPair(standardTarget(), analyticsTarget({ grammarHash: ' ' })), + /both report a non-empty grammar hash/ + ); + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget(), + analyticsTarget({ grammarHash: 'sha256:different-grammar' }) + ), + /report different values for grammar hash/ + ); +}); + +test('target schema validation runs before pair identity comparison', () => { + assert.throws( + () => + validatePrBuildTargetPair( + standardTarget({ schemaVersion: 1 }), + analyticsTarget() + ), + /standard PR-build target is invalid: Unsupported target schemaVersion/ + ); +}); + +test('paired backend reports require exact, usable query coverage', () => { + const base = { + standardTarget: standardTarget(), + analyticsTarget: analyticsTarget(), + standardReport: backendReport('standard'), + analyticsReport: backendReport('analytics'), + contractsDir: writeContractCorpus(), + }; + const result = validatePrBuildArtifacts(base); + assert.equal(result.expectedRows, 2); + + assert.throws( + () => + validatePrBuildArtifacts({ + ...base, + analyticsReport: backendReport('analytics', ['trigger']), + }), + /analytics PR-build backend report query coverage is incomplete.*test-rule::control/ + ); + assert.throws( + () => + validatePrBuildArtifacts({ + ...base, + standardReport: [ + ...backendReport('standard'), + { ...backendReport('standard')[0] }, + ], + }), + /duplicate backend report key/ + ); + + const errored = backendReport('analytics'); + errored[0] = { ...errored[0], outcome: 'error' }; + assert.throws( + () => validatePrBuildArtifacts({ ...base, analyticsReport: errored }), + /contains rows without an engine verdict: test-rule::trigger/ + ); + + const unobservedCoverageGap = backendReport('analytics'); + unobservedCoverageGap[0] = { + ...unobservedCoverageGap[0], + outcome: 'coverage-missing', + }; + delete unobservedCoverageGap[0].rejected; + assert.throws( + () => + validatePrBuildArtifacts({ + ...base, + analyticsReport: unobservedCoverageGap, + }), + /contains rows without an engine verdict: test-rule::trigger/ + ); + + const changedQuery = backendReport('analytics'); + changedQuery[0] = { ...changedQuery[0], query: 'source=different-index | head 0' }; + assert.throws( + () => validatePrBuildArtifacts({ ...base, analyticsReport: changedQuery }), + /executed different query text for test-rule::trigger/ + ); +}); + +test('the CLI reads and validates both PR-build artifact sets', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ppl-lint-target-pair-')); + tmpDirs.push(dir); + const standardFile = path.join(dir, 'standard.json'); + const analyticsFile = path.join(dir, 'analytics.json'); + const standardReportFile = path.join(dir, 'standard-report.json'); + const analyticsReportFile = path.join(dir, 'analytics-report.json'); + fs.writeFileSync(standardFile, JSON.stringify(standardTarget())); + fs.writeFileSync(analyticsFile, JSON.stringify(analyticsTarget())); + fs.writeFileSync(standardReportFile, JSON.stringify(backendReport('standard'))); + fs.writeFileSync(analyticsReportFile, JSON.stringify(backendReport('analytics'))); + + const result = spawnSync( + process.execPath, + [ + SCRIPT, + '--standard', + standardFile, + '--analytics', + analyticsFile, + '--standard-report', + standardReportFile, + '--analytics-report', + analyticsReportFile, + '--contracts', + writeContractCorpus(), + '--schedule', + 'nightly', + ], + { encoding: 'utf8' } + ); + + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /verified engine=3\.8\.0-SNAPSHOT/); + assert.match(result.stdout, /backends=standard,analytics/); + assert.match(result.stdout, /backendRows=2/); +}); diff --git a/scripts/ppl-lint/aggregate-versions.mjs b/scripts/ppl-lint/aggregate-versions.mjs index e5dd0510ca6..973b1f8728f 100644 --- a/scripts/ppl-lint/aggregate-versions.mjs +++ b/scripts/ppl-lint/aggregate-versions.mjs @@ -1382,7 +1382,11 @@ function main() { } const enforcedDrifts = drifts.filter((d) => d.blocking); const enforcedHoles = coverageHoles.filter((h) => h.blocking); - const enforcedInconclusive = inconclusive.filter((i) => i.enforced); + // `--all-rules` widens observation to the whole corpus. Semantic drift remains + // enforced only for default-error rules, but a missing detector row or backend + // verdict is an infrastructure failure for every rule we asked the run to + // observe. + const enforcedInconclusive = inconclusive.filter((i) => i.enforced || args.allRules); for (const row of matrix) { row.key = reportItemKey(row, 'matrix'); } diff --git a/scripts/ppl-lint/contract-schema.mjs b/scripts/ppl-lint/contract-schema.mjs index 5df505f07de..291f2f9c405 100644 --- a/scripts/ppl-lint/contract-schema.mjs +++ b/scripts/ppl-lint/contract-schema.mjs @@ -63,11 +63,17 @@ function assertBackendOracle(oracle, ruleId, executionBackend) { if (!Number.isInteger(oracle.httpStatus) || oracle.httpStatus < 100 || oracle.httpStatus > 599) { throw new TypeError(`${label}.httpStatus must be an integer from 100 through 599.`); } + if ((kind === 'result-shape' || kind === 'advisory') && oracle.httpStatus !== 200) { + throw new TypeError(`${label}.httpStatus must be 200.`); + } if (kind === 'rejection') { const body = requireObject(oracle.body, `${label}.body`); if (!Number.isInteger(body.status)) { throw new TypeError(`${label}.body.status must be an integer.`); } + if (body.status !== oracle.httpStatus) { + throw new TypeError(`${label}.httpStatus must equal ${label}.body.status.`); + } if (body.error !== undefined) { const error = requireObject(body.error, `${label}.body.error`); assertOptionalString(error.type, `${label}.body.error.type`); diff --git a/scripts/ppl-lint/validate-pr-build-targets.mjs b/scripts/ppl-lint/validate-pr-build-targets.mjs new file mode 100644 index 00000000000..b7944ed76d7 --- /dev/null +++ b/scripts/ppl-lint/validate-pr-build-targets.mjs @@ -0,0 +1,285 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs'; +import path from 'node:path'; + +import { + assertContractSchema, + classifyBackendReportRow, + indexBackendReport, + normalizeTarget, +} from './contract-schema.mjs'; + +function normalizeLabeledTarget(target, label) { + try { + return normalizeTarget(target); + } catch (error) { + throw new Error(`${label} target is invalid: ${error.message}`); + } +} + +function requireMatchingNonEmptyField(standard, analytics, field, label) { + if ( + typeof standard[field] !== 'string' || + standard[field].trim().length === 0 || + typeof analytics[field] !== 'string' || + analytics[field].trim().length === 0 + ) { + throw new Error( + `standard and analytics targets must both report a non-empty ${label}: ` + + `standard=${JSON.stringify(standard[field])}, analytics=${JSON.stringify(analytics[field])}` + ); + } + if (standard[field] !== analytics[field]) { + throw new Error( + `standard and analytics targets report different values for ${label}: ` + + `standard=${JSON.stringify(standard[field])}, analytics=${JSON.stringify(analytics[field])}` + ); + } +} + +export function validatePrBuildTargetPair(standardRaw, analyticsRaw) { + const standard = normalizeLabeledTarget(standardRaw, 'standard PR-build'); + const analytics = normalizeLabeledTarget(analyticsRaw, 'analytics PR-build'); + + if (standard.executionBackend !== 'standard') { + throw new Error( + `standard PR-build target executionBackend must be "standard", got ` + + `${JSON.stringify(standard.executionBackend)}` + ); + } + if (analytics.executionBackend !== 'analytics') { + throw new Error( + `analytics PR-build target executionBackend must be "analytics", got ` + + `${JSON.stringify(analytics.executionBackend)}` + ); + } + + requireMatchingNonEmptyField(standard, analytics, 'engineVersion', 'engine version'); + requireMatchingNonEmptyField(standard, analytics, 'sqlSha', 'SQL SHA'); + requireMatchingNonEmptyField(standard, analytics, 'grammarHash', 'grammar hash'); + + return { standard, analytics }; +} + +function expectedBackendReportKeys(contractsDir, schedule) { + if (schedule !== 'pr' && schedule !== 'nightly') { + throw new Error(`schedule must be "pr" or "nightly", got ${JSON.stringify(schedule)}`); + } + const manifest = readJson(path.join(contractsDir, 'manifest.json'), 'contract manifest'); + if ( + manifest === null || + typeof manifest !== 'object' || + Array.isArray(manifest) || + !Array.isArray(manifest.contracts) + ) { + throw new TypeError('contract manifest.contracts must be a JSON array'); + } + + const files = new Set(); + const ruleIds = new Set(); + const expectedRows = new Map(); + for (const file of manifest.contracts) { + if (typeof file !== 'string' || file.length === 0) { + throw new TypeError('contract manifest entries must be non-empty strings'); + } + if (files.has(file)) { + throw new Error(`contract manifest contains duplicate file ${JSON.stringify(file)}`); + } + files.add(file); + + const spec = readJson(path.join(contractsDir, file), `contract ${file}`); + assertContractSchema(spec); + if (ruleIds.has(spec.ruleId)) { + throw new Error(`contract manifest contains duplicate ruleId ${JSON.stringify(spec.ruleId)}`); + } + ruleIds.add(spec.ruleId); + if (schedule === 'pr' && (spec.schedule || 'pr') !== 'pr') { + continue; + } + if ( + spec.queries === null || + typeof spec.queries !== 'object' || + Array.isArray(spec.queries) || + Object.keys(spec.queries).length === 0 + ) { + throw new TypeError(`[${spec.ruleId}] contract.queries must be a non-empty JSON object`); + } + for (const queryName of Object.keys(spec.queries)) { + const key = `${spec.ruleId}::${queryName}`; + if (expectedRows.has(key)) { + throw new Error(`contract corpus contains duplicate query key ${JSON.stringify(key)}`); + } + const query = spec.queries[queryName]; + if (query === null || typeof query !== 'object' || Array.isArray(query)) { + throw new TypeError(`[${spec.ruleId}] query ${JSON.stringify(queryName)} must be an object`); + } + expectedRows.set(key, { role: query.role || 'trigger' }); + } + } + if (expectedRows.size === 0) { + throw new Error(`contract corpus selected no queries for schedule ${JSON.stringify(schedule)}`); + } + return expectedRows; +} + +function validateBackendReport(raw, target, label, expectedRows) { + let rows; + try { + rows = indexBackendReport(raw, target); + } catch (error) { + throw new Error(`${label} backend report is invalid: ${error.message}`); + } + + const missing = [...expectedRows.keys()].filter((key) => !rows.has(key)).sort(); + const extra = [...rows.keys()].filter((key) => !expectedRows.has(key)).sort(); + if (missing.length > 0 || extra.length > 0) { + const details = []; + if (missing.length > 0) details.push(`missing: ${missing.join(', ')}`); + if (extra.length > 0) details.push(`unexpected: ${extra.join(', ')}`); + throw new Error(`${label} backend report query coverage is incomplete (${details.join('; ')})`); + } + + const unusable = []; + for (const [key, row] of rows) { + const expected = expectedRows.get(key); + if (row.role !== expected.role) { + throw new Error( + `${label} backend report row ${key}.role must be ${JSON.stringify(expected.role)}, ` + + `got ${JSON.stringify(row.role)}` + ); + } + if (typeof row.query !== 'string' || row.query.length === 0) { + throw new Error(`${label} backend report row ${key}.query must be a non-empty string`); + } + const status = classifyBackendReportRow(row).status; + if ( + status === 'error' || + (status === 'coverage-missing' && typeof row.rejected !== 'boolean') + ) { + unusable.push(key); + } + } + if (unusable.length > 0) { + throw new Error( + `${label} backend report contains rows without an engine verdict: ${unusable.sort().join(', ')}` + ); + } + return rows; +} + +export function validatePrBuildArtifacts({ + standardTarget, + analyticsTarget, + standardReport, + analyticsReport, + contractsDir, + schedule = 'nightly', +}) { + const pair = validatePrBuildTargetPair(standardTarget, analyticsTarget); + const expectedRows = expectedBackendReportKeys(contractsDir, schedule); + const standardRows = validateBackendReport( + standardReport, + pair.standard, + 'standard PR-build', + expectedRows + ); + const analyticsRows = validateBackendReport( + analyticsReport, + pair.analytics, + 'analytics PR-build', + expectedRows + ); + for (const key of expectedRows.keys()) { + const standard = standardRows.get(key); + const analytics = analyticsRows.get(key); + if (standard.query !== analytics.query) { + throw new Error( + `standard and analytics backend reports executed different query text for ${key}: ` + + `standard=${JSON.stringify(standard.query)}, analytics=${JSON.stringify(analytics.query)}` + ); + } + } + return { ...pair, expectedRows: expectedRows.size, standardRows, analyticsRows }; +} + +function parseArgs(argv) { + const options = new Map([ + ['--standard', 'standard'], + ['--analytics', 'analytics'], + ['--standard-report', 'standardReport'], + ['--analytics-report', 'analyticsReport'], + ['--contracts', 'contracts'], + ['--schedule', 'schedule'], + ]); + const args = { schedule: 'nightly' }; + const seen = new Set(); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const key = options.get(arg); + if (!key) { + throw new Error(`unknown argument ${JSON.stringify(arg)}`); + } + const value = argv[++i]; + if (!value) { + throw new Error(`${arg} requires a value`); + } + if (seen.has(key)) { + throw new Error(`${arg} may be specified only once`); + } + seen.add(key); + args[key] = value; + } + for (const key of [ + 'standard', + 'analytics', + 'standardReport', + 'analyticsReport', + 'contracts', + ]) { + if (!args[key]) { + throw new Error(`${[...options].find(([, value]) => value === key)[0]} is required`); + } + } + return args; +} + +function readJson(file, label) { + try { + return JSON.parse(fs.readFileSync(file, 'utf8')); + } catch (error) { + throw new Error(`could not read ${label} ${file}: ${error.message}`); + } +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const { standard, analytics, expectedRows } = validatePrBuildArtifacts({ + standardTarget: readJson(args.standard, 'standard PR-build target'), + analyticsTarget: readJson(args.analytics, 'analytics PR-build target'), + standardReport: readJson(args.standardReport, 'standard PR-build backend report'), + analyticsReport: readJson(args.analyticsReport, 'analytics PR-build backend report'), + contractsDir: args.contracts, + schedule: args.schedule, + }); + // eslint-disable-next-line no-console + console.log( + `[ppl-lint-target-pair] verified engine=${standard.engineVersion} ` + + `sqlSha=${standard.sqlSha} grammarHash=${standard.grammarHash} ` + + `backends=${standard.executionBackend},${analytics.executionBackend} ` + + `backendRows=${expectedRows}` + ); +} + +if (process.argv[1] && process.argv[1].endsWith('validate-pr-build-targets.mjs')) { + try { + main(); + } catch (error) { + // eslint-disable-next-line no-console + console.error(`[ppl-lint-target-pair] FATAL: ${error.message}`); + process.exitCode = 2; + } +} From 5576c8a846d897fd8b2aacd75ff4e03e0f090bda Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 14:22:55 -0700 Subject: [PATCH 37/39] fix(ci): resolve analytics lint contracts Signed-off-by: Hanyu Wei --- integ-test/build.gradle | 1 + 1 file changed, 1 insertion(+) diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 9452087ace6..fcceb74a107 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -610,6 +610,7 @@ task analyticsEnginePplLintIT(type: RestIntegTestTask) { systemProperty 'ppl.lint.execution_backend', 'analytics' systemProperty 'ppl.lint.analytics.stack.source', featureBuildBase systemProperty 'tests.security.manager', 'false' + systemProperty 'project.root', project.projectDir.absolutePath filter { includeTestsMatching 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' From e44dfdba66106618ee2cced81b4b038c20373aed Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 14:59:30 -0700 Subject: [PATCH 38/39] fix(ci): support append-only analytics fixtures Signed-off-by: Hanyu Wei --- .../remote/PplLintRuleValidationIT.java | 122 +++++++++++- .../sql/legacy/AnalyticsFieldStripTests.java | 59 +++++- .../org/opensearch/sql/legacy/TestUtils.java | 89 +++++++-- .../contracts/flat-object-subfield.spec.json | 184 ++++++++++++------ scripts/ppl-lint/run-frontend-contract.mjs | 6 +- 5 files changed, 367 insertions(+), 93 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index f873e4c0021..e8dbafc42a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -169,6 +169,8 @@ public void init() throws Exception { // super.init() aborted partway, so redo the part that is version-independent. increaseMaxCompilationsRate(); } + clusterVersion = fetchClusterVersion(); + // Fall through to fixture seeding either way. // Seed the union of every index every scheduled contract needs, once. for (String indexEnum : requiredIndexEnums()) { @@ -194,7 +196,6 @@ public void init() throws Exception { "[ppl-lint] could not seed index " + indexEnum + " on this engine: " + e.getMessage()); } } - clusterVersion = fetchClusterVersion(); } @Test @@ -221,6 +222,8 @@ public void testValidatesLintRuleContracts() throws IOException { String ruleId = contract.getString("ruleId"); runContract(contract, ruleId, failures, report); } + } else { + recordUnattestedRouteContracts(contracts, report); } try { @@ -365,6 +368,52 @@ && recordEnforcementCoverageGaps( } } + /** + * Route attestation failure prevents query execution, but it must not produce a misleadingly + * empty report. Emit one non-verdict row per query; explicit, complete non-applicable rows remain + * non-applicable because they do not depend on the unavailable fixture. + */ + private void recordUnattestedRouteContracts(List contracts, JSONArray report) { + for (JSONObject contract : contracts) { + String ruleId = contract.getString("ruleId"); + String index = contract.getString("index"); + JSONObject queries = contract.getJSONObject("queries"); + JSONObject fixture = contract.optJSONObject("backendFixture"); + List matches = + matchingExpectations( + contract.getJSONArray("expectations"), fixtureCalciteEnabled(fixture)); + JSONObject expectedQueries = + matches.size() == 1 ? matches.get(0).optJSONObject("queries") : null; + + for (String queryName : queries.keySet()) { + JSONObject queryDef = queries.getJSONObject(queryName); + String role = queryDef.optString("role", "trigger"); + String query = queryDef.getString("query").replace("{{index}}", index); + JSONObject expected = + expectedQueries == null ? null : expectedQueries.optJSONObject(queryName); + JSONObject backend = null; + int schemaVersion = contract.optInt("schemaVersion"); + if (expected != null && (schemaVersion == 3 || schemaVersion == 4)) { + try { + backend = resolveBackendOracle(schemaVersion, expected); + } catch (RuntimeException ignored) { + // Malformed contracts are still failures; keep this report row as a non-verdict. + } + } + JSONObject entry = reportEntry(ruleId, queryName, role, query, "route-attestation-failed"); + if (isCompleteNotApplicableOracle(backend)) { + entry.put("kind", "not-applicable"); + recordNotApplicable(ruleId, queryName, backend, entry, report); + } else { + report.put( + entry + .put("outcome", "error") + .put("error", "analytics route attestation failed before contract execution")); + } + } + } + } + /** * The first index fixture this contract needs that failed to seed, or null when every index it * declares is present. Only ever non-null in observe-only mode, where a seeding failure is @@ -1283,9 +1332,7 @@ private void verifyAnalyticsFixtureIndices() throws IOException { requireIndexSetting( indexName, settings, "index.number_of_shards", Integer.toString(expectedShards)); - Response countResponse = - client().performRequest(new Request("GET", "/" + indexName + "/_count")); - long count = new JSONObject(getResponseBody(countResponse, true)).getLong("count"); + long count = analyticsDocumentCount(indexName); documentCounts.put(indexName, count); fixtureEvidence.put("documentCount", count); requireAttestation( @@ -1294,6 +1341,18 @@ private void verifyAnalyticsFixtureIndices() throws IOException { } } + private long analyticsDocumentCount(String indexName) throws IOException { + JSONObject response = + executeQuery("source=" + indexName + " | stats count() as document_count"); + JSONArray rows = response.getJSONArray("datarows"); + requireAttestation(rows.length() == 1, "fixture " + indexName + " count returned " + rows); + JSONArray row = rows.getJSONArray(0); + requireAttestation( + row.length() == 1 && row.get(0) instanceof Number, + "fixture " + indexName + " count did not return one numeric value: " + rows); + return ((Number) row.get(0)).longValue(); + } + private String canonicalJson(Object value) { if (value == null || value == JSONObject.NULL) { return "null"; @@ -1803,10 +1862,19 @@ private List manifestContractNames() throws IOException { return names; } - /** Union of index enums required by the contracts scheduled to run this session. */ + /** + * Union of index enums required by the contracts scheduled to run this session. + * + *

      A schema-v4 contract whose selected analytics oracle marks every query explicitly + * non-applicable does not need its unrepresentable fixture. Missing or malformed oracles remain + * fixture-requiring so they cannot turn into an implicit skip. + */ private Set requiredIndexEnums() throws IOException { Set indices = new LinkedHashSet<>(); for (JSONObject contract : loadScheduledContracts()) { + if (!contractRequiresFixture(contract)) { + continue; + } JSONObject fixture = contract.optJSONObject("backendFixture"); if (fixture == null) { continue; @@ -1825,6 +1893,50 @@ private Set requiredIndexEnums() throws IOException { return indices; } + private boolean contractRequiresFixture(JSONObject contract) { + if (executionBackend != ExecutionBackend.ANALYTICS || contract.optInt("schemaVersion") != 4) { + return true; + } + + JSONObject fixture = contract.optJSONObject("backendFixture"); + List matches = + matchingExpectations(contract.getJSONArray("expectations"), fixtureCalciteEnabled(fixture)); + if (matches.size() != 1) { + return true; + } + + JSONObject declaredQueries = contract.optJSONObject("queries"); + JSONObject expectedQueries = matches.get(0).optJSONObject("queries"); + if (declaredQueries == null + || declaredQueries.length() == 0 + || expectedQueries == null + || !declaredQueries.keySet().equals(expectedQueries.keySet())) { + return true; + } + + for (String queryName : declaredQueries.keySet()) { + JSONObject expected = expectedQueries.optJSONObject(queryName); + JSONObject backend = expected == null ? null : resolveBackendOracle(4, expected); + if (!isCompleteNotApplicableOracle(backend)) { + return true; + } + } + return false; + } + + private boolean isCompleteNotApplicableOracle(JSONObject backend) { + return backend != null + && "not-applicable".equals(backend.optString("kind")) + && hasNonBlankString(backend, "reason") + && hasNonBlankString(backend, "owner") + && hasNonBlankString(backend, "issue"); + } + + private boolean hasNonBlankString(JSONObject object, String key) { + Object value = object.opt(key); + return value instanceof String && !((String) value).trim().isEmpty(); + } + private JSONObject loadContractFile(String resourcePath) throws IOException { String path = TestUtils.getResourceFilePath(resourcePath); return new JSONObject(new String(Files.readAllBytes(Paths.get(path)))); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java index 36d2c4ed82a..1db2e38da9e 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -7,6 +7,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import java.util.List; @@ -168,7 +169,7 @@ public void mappingStrip_noopWhenDisabled() { public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { enable(); String bulk = - "{\"index\":{\"_id\":\"1\"}}\n" + "{\"index\":{\"_index\":\"one\",\"_id\":\"1\",\"routing\":\"r1\"}}\n" + "{\"keep_text\":\"x\",\"geo_point_value\":{\"lat\":1,\"lon\":2},\"geo_shape_value\":\"POINT(1" + " 2)\"}\n" + "{\"index\":{\"_id\":\"2\"}}\n" @@ -178,9 +179,12 @@ public void bulkStrip_removesDroppedPathsFromSourceLinesOnly() { bulk, Set.of(path("geo_point_value"), path("geo_shape_value"), path("nested_value"))); String[] lines = out.split("\n"); - // action lines untouched - assertTrue(lines[0].contains("\"index\"")); - assertTrue(lines[2].contains("\"index\"")); + // Append-only analytics indices require generated IDs, but all other action metadata remains. + JSONObject firstAction = new JSONObject(lines[0]).getJSONObject("index"); + assertFalse(firstAction.has("_id")); + assertEquals("one", firstAction.getString("_index")); + assertEquals("r1", firstAction.getString("routing")); + assertFalse(new JSONObject(lines[2]).getJSONObject("index").has("_id")); // source lines stripped, supported field retained JSONObject doc1 = new JSONObject(lines[1]); assertTrue(doc1.has("keep_text")); @@ -213,12 +217,51 @@ public void bulkStrip_leavesUntouchedSourceLinesByteForByte() { } @Test - public void bulkStrip_noopWhenDisabledOrEmptyDropSet() { - String bulk = "{\"index\":{}}\n{\"geo_point_value\":{\"lat\":1}}\n"; + public void bulkStrip_emptyDropSet_onlyRemovesAnalyticsCustomIds() { + String indexSource = "{\"index\":\"source-value\",\"spacing\": 2}"; + String createSource = "{\"delete\":\"also-a-source-value\"}"; + String bulk = + "{\"index\":{\"_index\":\"fixture\",\"_id\":\"1\"}}\n" + + indexSource + + "\n" + + "{\"create\":{\"_index\":\"fixture\",\"_id\":\"2\",\"routing\":\"r2\"}}\n" + + createSource + + "\n"; // disabled -> unchanged even with a drop set assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of(path("geo_point_value")))); - // enabled but empty drop set -> unchanged + + // enabled with no dropped fields -> generated IDs for append-only writes, source unchanged enable(); - assertEquals(bulk, AnalyticsIndexConfig.stripBulkFields(bulk, Set.of())); + String out = AnalyticsIndexConfig.stripBulkFields(bulk, Set.of()); + String[] lines = out.split("\n", -1); + JSONObject index = new JSONObject(lines[0]).getJSONObject("index"); + assertFalse(index.has("_id")); + assertEquals("fixture", index.getString("_index")); + assertEquals(indexSource, lines[1]); + JSONObject create = new JSONObject(lines[2]).getJSONObject("create"); + assertEquals("2", create.getString("_id")); + assertEquals("fixture", create.getString("_index")); + assertEquals("r2", create.getString("routing")); + assertEquals(createSource, lines[3]); + // split(..., -1) proves the original terminal newline survived. + assertEquals("", lines[4]); + } + + @Test + public void bulkStrip_rejectsActionsThatAppendOnlyStorageCannotRepresent() { + enable(); + IllegalArgumentException update = + assertThrows( + IllegalArgumentException.class, + () -> + AnalyticsIndexConfig.stripBulkFields( + "{\"update\":{\"_id\":\"1\"}}\n{\"doc\":{\"value\":1}}\n", Set.of())); + assertTrue(update.getMessage().contains("does not support update")); + + IllegalArgumentException delete = + assertThrows( + IllegalArgumentException.class, + () -> AnalyticsIndexConfig.stripBulkFields("{\"delete\":{\"_id\":\"1\"}}\n", Set.of())); + assertTrue(delete.getMessage().contains("does not support delete")); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 198527d1efc..2a5b996dea1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -248,41 +248,64 @@ private static void collectAndRemoveUnsupported( } /** - * Strip the given dropped paths from every source document of a bulk NDJSON payload. - * Bulk format alternates an action line ({@code {"index":{...}}}) with a source line; only - * source lines (those without a bulk action key) are rewritten. No-op when disabled or {@code - * droppedPaths} is empty. + * Prepare a bulk NDJSON payload for an analytics-engine append-only index. + * + *

      Custom document IDs are not supported for {@code index} operations when {@code + * index.append_only.enabled} is active, so {@code _id} is removed from that action metadata + * while preserving metadata such as {@code _index} and {@code routing}. Create actions retain + * their semantics; update/delete actions fail locally because they are incompatible with + * append-only storage. + * + *

      The given dropped paths are also removed from every source document. Bulk format + * alternates an action line ({@code {"index":{...}}}) with a source line; only source lines + * (those without a bulk action key) have mapped fields removed. No-op when analytics mode is + * disabled. * *

      Each path is removed recursively: it descends through nested objects and arrays of * objects (so a {@code nested}/object array has the field stripped from every element), * leaving unaffected siblings intact. A source line is re-serialized only when removing a - * path actually changed it; every other line (action lines and docs that never had the - * dropped path) is appended byte-for-byte unchanged, so untouched docs match the fixture - * exactly. + * path actually changed it. Action lines are re-serialized only when removing {@code _id}; + * every other line is appended byte-for-byte unchanged. */ static String stripBulkFields(String bulkBody, Set> droppedPaths) { - if (!isEnabled() || droppedPaths.isEmpty()) { + if (!isEnabled()) { return bulkBody; } String[] lines = bulkBody.split("\n", -1); StringBuilder out = new StringBuilder(bulkBody.length()); + boolean expectSource = false; for (int i = 0; i < lines.length; i++) { String line = lines[i]; String trimmed = line.trim(); - if (!trimmed.isEmpty() && trimmed.charAt(0) == '{') { - JSONObject doc = new JSONObject(trimmed); - boolean isActionLine = - doc.has("index") || doc.has("create") || doc.has("update") || doc.has("delete"); - if (!isActionLine) { + boolean terminalNewline = i == lines.length - 1 && trimmed.isEmpty(); + if (!terminalNewline) { + if (trimmed.isEmpty()) { + throw new IllegalArgumentException( + "analytics bulk payload contains a blank NDJSON line"); + } + + JSONObject json = new JSONObject(trimmed); + if (expectSource) { boolean removedAny = false; for (List path : droppedPaths) { - removedAny |= removePath(doc, path, 0); + removedAny |= removePath(json, path, 0); } // Only rewrite the line if we actually removed something; otherwise leave it verbatim // so untouched docs stay byte-for-byte identical to the fixture. if (removedAny) { - line = doc.toString(); + line = json.toString(); + } + expectSource = false; + } else { + String operation = bulkOperation(json); + if ("update".equals(operation) || "delete".equals(operation)) { + throw new IllegalArgumentException( + "analytics append-only bulk payload does not support " + operation + " actions"); } + if ("index".equals(operation) && removeCustomDocumentId(json, operation)) { + line = json.toString(); + } + expectSource = true; } } out.append(line); @@ -290,9 +313,40 @@ static String stripBulkFields(String bulkBody, Set> droppedPaths) { out.append('\n'); } } + if (expectSource) { + throw new IllegalArgumentException( + "analytics bulk payload ended before the final action's source document"); + } return out.toString(); } + private static String bulkOperation(JSONObject action) { + List operations = + List.of("index", "create", "update", "delete").stream() + .filter(action::has) + .collect(Collectors.toList()); + if (operations.size() != 1 || action.length() != 1) { + throw new IllegalArgumentException( + "analytics bulk action line must contain exactly one index/create/update/delete" + + " action"); + } + String operation = operations.get(0); + if (!(action.opt(operation) instanceof JSONObject)) { + throw new IllegalArgumentException( + "analytics bulk " + operation + " action metadata must be an object"); + } + return operation; + } + + private static boolean removeCustomDocumentId(JSONObject action, String operation) { + JSONObject metadata = action.optJSONObject(operation); + if (metadata == null || !metadata.has("_id")) { + return false; + } + metadata.remove("_id"); + return true; + } + /** * Remove {@code path[idx..]} from {@code node}, descending through objects and arrays of * objects. Returns true if anything was removed. At the last path part the key is deleted from @@ -429,8 +483,9 @@ public static void loadDataByRestClient( /** * Same as {@link #loadDataByRestClient(RestClient, String, String)} but strips {@code * droppedPaths} (the exact field paths removed from the mapping on the analytics-engine route) - * from every bulk source doc, so the index mapping and the data agree. When AE is disabled or - * {@code droppedPaths} is empty this is byte-for-byte identical to the 3-arg form. + * from every bulk source doc, so the index mapping and the data agree. Analytics append-only + * {@code index} operations also discard custom document IDs. When analytics mode is disabled this + * is byte-for-byte identical to the 3-arg form. */ public static void loadDataByRestClient( RestClient client, String indexName, String dataSetFilePath, Set> droppedPaths) diff --git a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json index 440c55e27ac..0ba5c95be35 100644 --- a/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json +++ b/integ-test/src/test/resources/ppl-lint/contracts/flat-object-subfield.spec.json @@ -1,5 +1,5 @@ { - "schemaVersion": 3, + "schemaVersion": 4, "ruleId": "flat-object-subfield", "grammarSurface": "runtime-bundle", "schedule": "pr", @@ -7,7 +7,7 @@ "qualifiedName", "wcQualifiedName" ], - "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true).", + "notes": "Live-verified on OpenSearch 3.8 with Calcite on: a flat_object field cannot be referenced by PPL at all. BOTH a dotted subfield (`fields attributes.service`) AND the bare root (`fields attributes`) fail with IllegalArgumentException 'Field [...] not found.', and the same holds in a where clause. NOTE the rejection reason is byte-identical to the one field-validation produces for a genuinely absent field, so the backend reason alone cannot attribute a diagnostic to a rule — attribution comes from the detector's ruleId, which is why every case here pins detectorCount for THIS ruleId only. The detector self-suppresses without a typeMap, hence the deriveFromMapping block below (needsContext: true). The analytics feature build cannot create flat_object in composite/Parquet storage, so every analytics backend oracle is explicitly non-applicable while the frontend detector assertions still run.", "wiring": { "detector": "flat-object-subfield", "enabled": true, @@ -61,55 +61,87 @@ "flat-object-dotted-subfield": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-bare-root": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-in-where": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Invalid Query" + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Invalid Query" + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "non-flat-field-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } } @@ -122,55 +154,87 @@ "flat-object-dotted-subfield": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [attributes.service] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-bare-root": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [attributes] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes] not found." + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "flat-object-in-where": { "detectorCount": 1, "severity": "error", - "backend": { - "kind": "rejection", - "httpStatus": 400, - "body": { - "status": 400, - "error": { - "type": "IllegalArgumentException", - "reason": "Field [attributes.service] not found." + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Field [attributes.service] not found." + } } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } }, "non-flat-field-control": { "detectorCount": 0, - "backend": { - "kind": "result-shape", - "httpStatus": 200, - "expect": { - "datarowsNonEmpty": true + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "not-applicable", + "reason": "flat_object fields cannot be represented by the analytics composite/Parquet fixture", + "owner": "@Hanyu-W", + "issue": "https://github.com/Hanyu-W/sql/pull/3" } } } diff --git a/scripts/ppl-lint/run-frontend-contract.mjs b/scripts/ppl-lint/run-frontend-contract.mjs index 3606ee6510f..7bd7de2ce61 100644 --- a/scripts/ppl-lint/run-frontend-contract.mjs +++ b/scripts/ppl-lint/run-frontend-contract.mjs @@ -807,9 +807,9 @@ function main() { } if (oracleSelection.status === 'not-applicable') { - resultEntry.outcome = 'not-applicable'; - resultEntry.reason = oracleSelection.reason; - resultEntry.notApplicable = oracleSelection.reason; + // Only the backend fixture is non-applicable. The detector still ran above and its + // count/severity/message assertions remain ordinary, comparable frontend evidence. + resultEntry.backendOracleReason = oracleSelection.reason; } else if (oracleSelection.status === 'coverage-missing') { resultEntry.outcome = 'coverage-missing'; resultEntry.coverage = 'missing'; From 288eb5add5a7034d84b34764e36784dfe3b29be0 Mon Sep 17 00:00:00 2001 From: Hanyu Wei Date: Tue, 28 Jul 2026 15:01:39 -0700 Subject: [PATCH 39/39] test(analytics): preserve bulk separators Signed-off-by: Hanyu Wei --- .../sql/legacy/AnalyticsFieldStripTests.java | 9 ++-- .../org/opensearch/sql/legacy/TestUtils.java | 50 ++++++++++--------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java index 1db2e38da9e..0fcedfee093 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/AnalyticsFieldStripTests.java @@ -223,7 +223,7 @@ public void bulkStrip_emptyDropSet_onlyRemovesAnalyticsCustomIds() { String bulk = "{\"index\":{\"_index\":\"fixture\",\"_id\":\"1\"}}\n" + indexSource - + "\n" + + "\n\n" + "{\"create\":{\"_index\":\"fixture\",\"_id\":\"2\",\"routing\":\"r2\"}}\n" + createSource + "\n"; @@ -238,13 +238,14 @@ public void bulkStrip_emptyDropSet_onlyRemovesAnalyticsCustomIds() { assertFalse(index.has("_id")); assertEquals("fixture", index.getString("_index")); assertEquals(indexSource, lines[1]); - JSONObject create = new JSONObject(lines[2]).getJSONObject("create"); + assertEquals("", lines[2]); + JSONObject create = new JSONObject(lines[3]).getJSONObject("create"); assertEquals("2", create.getString("_id")); assertEquals("fixture", create.getString("_index")); assertEquals("r2", create.getString("routing")); - assertEquals(createSource, lines[3]); + assertEquals(createSource, lines[4]); // split(..., -1) proves the original terminal newline survived. - assertEquals("", lines[4]); + assertEquals("", lines[5]); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 2a5b996dea1..25385201e19 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -280,32 +280,36 @@ static String stripBulkFields(String bulkBody, Set> droppedPaths) { boolean terminalNewline = i == lines.length - 1 && trimmed.isEmpty(); if (!terminalNewline) { if (trimmed.isEmpty()) { - throw new IllegalArgumentException( - "analytics bulk payload contains a blank NDJSON line"); - } - - JSONObject json = new JSONObject(trimmed); - if (expectSource) { - boolean removedAny = false; - for (List path : droppedPaths) { - removedAny |= removePath(json, path, 0); - } - // Only rewrite the line if we actually removed something; otherwise leave it verbatim - // so untouched docs stay byte-for-byte identical to the fixture. - if (removedAny) { - line = json.toString(); - } - expectSource = false; - } else { - String operation = bulkOperation(json); - if ("update".equals(operation) || "delete".equals(operation)) { + if (expectSource) { throw new IllegalArgumentException( - "analytics append-only bulk payload does not support " + operation + " actions"); + "analytics bulk action is missing its source document"); } - if ("index".equals(operation) && removeCustomDocumentId(json, operation)) { - line = json.toString(); + } else { + JSONObject json = new JSONObject(trimmed); + if (expectSource) { + boolean removedAny = false; + for (List path : droppedPaths) { + removedAny |= removePath(json, path, 0); + } + // Only rewrite the line if we actually removed something; otherwise leave it + // verbatim so untouched docs stay byte-for-byte identical to the fixture. + if (removedAny) { + line = json.toString(); + } + expectSource = false; + } else { + String operation = bulkOperation(json); + if ("update".equals(operation) || "delete".equals(operation)) { + throw new IllegalArgumentException( + "analytics append-only bulk payload does not support " + + operation + + " actions"); + } + if ("index".equals(operation) && removeCustomDocumentId(json, operation)) { + line = json.toString(); + } + expectSource = true; } - expectSource = true; } } out.append(line);