diff --git a/.github/workflows/ppl-lint-multiversion-validation.yml b/.github/workflows/ppl-lint-multiversion-validation.yml new file mode 100644 index 00000000000..de047b07569 --- /dev/null +++ b/.github/workflows/ppl-lint-multiversion-validation.yml @@ -0,0 +1,1181 @@ +name: PPL lint multi-version validation + +# Multi-version companion to ppl-lint-rule-validation.yml. +# +# The sibling workflow answers "do the OSD PPL lint detectors and THIS engine +# build agree?". It validates one engine: the one built from the PR. That leaves +# the failure mode that actually reaches users unguarded — a lint rule ships to +# everyone, but each user runs it against whatever engine version their cluster +# happens to be. A rule that is correct on main can be a false positive on 3.6 or +# a false negative on 3.7, and nothing notices. +# +# This workflow validates every DEFAULT-ERROR rule (enabled: true + severity: +# error in OSD's rules_catalog.json) against SEVERAL released engine versions +# plus the PR build, and — when a rule disagrees with any of them — says what to +# change in the linter rather than only that a count was wrong. +# +# Why default-error only: an error-severity rule is one the user cannot opt out +# of and which marks their query as broken. A wrong error is the most expensive +# possible lint defect, so that set gets the multi-version treatment first. +# Warning/info rules stay on the single-version check. The set is not hand-copied: +# the detector run records the catalog's default-error census, and the aggregate +# step fails if a rule in that census has no contract file (see manifest.json's +# `defaultError` note). +# +# Shape — a per-version matrix of observation legs, then one aggregation: +# +# observe (matrix: 3.6.0, 3.7.0, pr-build) ──▶ aggregate ──▶ drift report +# +# Each leg produces the SAME four artifacts the single-version workflow already +# defines (ppl-grammar-bundle.json, target.json, backend-report.json, +# detector-report.json), so this workflow adds no new producer format — only the +# per-version fan-out and the cross-version comparison. +# +# Released legs run the official distribution image, which bundles the matching +# opensearch-sql plugin (verified against opensearch-build's release manifests), +# so no old branch has to be built. The `pr-build` leg is the same Gradle test +# cluster the sibling workflow uses. +# +# Engine floor: 3.6.0. GET /_plugins/_ppl/_grammar landed in #5162 (`fe95703b5`), +# which is an ancestor of the 3.6 release branch but NOT of 3.5 — a 3.5 leg could +# not export a candidate grammar bundle, so the detector half would have nothing +# to lint against. Raise `ENGINE_VERSIONS` as older versions leave support. +# +# Non-enforcing on purpose, for now: it reports and uploads, and the required +# check stays the sibling workflow's `validation-result`. Promoting this to +# required needs a green baseline across the whole matrix first (a rule that has +# quietly drifted on 3.6 would otherwise block every unrelated PR on day one). + +on: + # Nightly is the primary schedule: the matrix pulls three engine images, so it + # is too slow to sit on every push. + schedule: + - cron: '30 10 * * *' + # Run on PRs that touch the contract corpus or this machinery, where the whole + # point is to see the multi-version effect of the change. + pull_request: + paths: + - 'integ-test/build.gradle' + - 'integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java' + - 'integ-test/src/test/resources/ppl-lint/**' + - 'scripts/ppl-lint-rule-validation.sh' + - 'scripts/ppl-lint/**' + - '.github/workflows/ppl-lint-multiversion-validation.yml' + workflow_dispatch: + inputs: + osd_repo: + description: OSD repository to check out. Defaults to opensearch-project/OpenSearch-Dashboards. + required: false + type: string + osd_ref: + description: OSD commit or branch whose detectors are validated. + required: false + type: string + engine_versions: + description: 'JSON array of released engine versions to validate, e.g. ["3.6.0","3.7.0"]. The PR build is always added.' + required: false + type: string + compiled_versions: + description: 'JSON array of engine versions to validate on the compiled-simplified surface, e.g. ["2.19.0"]. Use "[]" to skip them.' + required: false + type: string + +permissions: + contents: read + +concurrency: + group: ppl-lint-multiversion-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + # Released engine versions to validate on the RUNTIME-BUNDLE surface. Each must + # be >= 3.6.0 (the _grammar endpoint floor) and must have a published + # distribution image. + # Latest patch of each line (3.6.0 and 3.7.0 ARE the latest patches today; bump + # them when 3.6.1 / 3.7.1 publish, and never pin `.0` once a newer patch + # exists — that would validate an engine no user runs). + ENGINE_VERSIONS: '["3.6.0","3.7.0"]' + # Released engine versions to validate on the COMPILED-SIMPLIFIED surface. + # + # These engines cannot export a grammar bundle (GET /_plugins/_ppl/_grammar + # landed in 3.6), so the runtime surface cannot reach them at all. But the + # compiled surface has no such floor: it lints with OSD's own checked-in grammar, + # which is exactly what a user gets when no bundle is available — including every + # user on an engine below 3.6. Those legs still run the real contract queries + # against the real engine, so the backend half of the differential is genuine. + # + # Only contracts declaring `grammarSurface: "both"` are scored here; the rest are + # reported not-applicable. Nightly only — see the `compiled_versions` input to + # run one ad hoc. + # + # Always the LATEST PATCH of each line, never `.0`. A user on 2.19 is on + # 2.19.6, so validating 2.19.0 tests an engine nobody runs and attributes any + # bug fixed in between to the whole line. + COMPILED_ENGINE_VERSIONS: '["2.19.6","3.0.0","3.5.0"]' + +jobs: + # Same reusable workflow + pinned SHA the sibling SQL workflows use, so a + # dependabot bump moves one set of action versions rather than two. + Get-CI-Image-Tag: + uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@761e093b8c1349cc07f21c1d681d3b30bf9e1999 # main + with: + product: opensearch + + # Resolve the matrix and the OSD target once, so every leg and the aggregate + # step agree on exactly what is being validated. + plan: + name: Plan matrix + runs-on: ubuntu-latest + outputs: + released: ${{ steps.plan.outputs.released }} + compiled: ${{ steps.plan.outputs.compiled }} + discovery_engine: ${{ steps.plan.outputs.discovery_engine }} + osd_repo: ${{ steps.plan.outputs.osd_repo }} + osd_ref: ${{ steps.plan.outputs.osd_ref }} + steps: + - name: Resolve engine versions and OSD target + id: plan + env: + REQUESTED_VERSIONS: ${{ inputs.engine_versions }} + DEFAULT_VERSIONS: ${{ env.ENGINE_VERSIONS }} + REQUESTED_COMPILED: ${{ inputs.compiled_versions }} + DEFAULT_COMPILED: ${{ env.COMPILED_ENGINE_VERSIONS }} + EVENT_NAME: ${{ github.event_name }} + REQUESTED_REPO: ${{ inputs.osd_repo }} + REQUESTED_REF: ${{ inputs.osd_ref }} + VAR_REPO: ${{ vars.OSD_REPO }} + VAR_REF: ${{ vars.OSD_REF }} + run: | + set -euo pipefail + released="${REQUESTED_VERSIONS:-$DEFAULT_VERSIONS}" + # Fail loudly on a malformed override rather than silently validating + # an empty matrix (which would look like a pass). + echo "$released" | python3 -c " + import json,sys + v=json.load(sys.stdin) + assert isinstance(v,list) and v, 'engine_versions must be a non-empty JSON array' + for item in v: + assert isinstance(item,str), 'engine_versions entries must be strings' + " + echo "released=$released" >> "$GITHUB_OUTPUT" + + # Compiled-surface legs add three more engine images, so they run on the + # nightly schedule (and on an explicit dispatch), not on every PR that + # touches the corpus. An explicit input always wins, including "[]". + if [ -n "${REQUESTED_COMPILED:-}" ]; then + compiled="$REQUESTED_COMPILED" + elif [ "$EVENT_NAME" = "pull_request" ]; then + compiled='[]' + else + compiled="$DEFAULT_COMPILED" + fi + # An EMPTY list is legitimate here (unlike engine_versions): it means "skip + # the compiled surface this run". Still reject a non-list. + echo "$compiled" | python3 -c " + import json,sys + v=json.load(sys.stdin) + assert isinstance(v,list), 'compiled_versions must be a JSON array' + for item in v: + assert isinstance(item,str), 'compiled_versions entries must be strings' + " + echo "compiled=$compiled" >> "$GITHUB_OUTPUT" + echo "Compiled-surface legs: $compiled" >> "$GITHUB_STEP_SUMMARY" + + # Discovery runs against ONE engine — the newest released version in the + # matrix. It is a lead-generator, not a version-drift check, so paying for + # a full matrix would multiply cost without adding signal: a false positive + # found on the newest engine is the one users hit soonest, and per-version + # differences are already the enforced corpus's job. + discovery_engine=$(echo "$released" | python3 -c " + import json,sys + v=json.load(sys.stdin) + # Newest by semver, not list order, so a reordered matrix cannot silently + # point discovery at an old engine. + def key(s): + parts=[int(p) for p in s.split('-')[0].split('.') if p.isdigit()] + return parts + [0]*(3-len(parts)) + print(sorted(v,key=key)[-1]) + ") + echo "discovery_engine=$discovery_engine" >> "$GITHUB_OUTPUT" + echo "Discovery engine: \`$discovery_engine\`" >> "$GITHUB_STEP_SUMMARY" + # Same precedence as the sibling workflow: dispatch input, then repo + # variable, then the canonical upstream default. + echo "osd_repo=${REQUESTED_REPO:-${VAR_REPO:-opensearch-project/OpenSearch-Dashboards}}" >> "$GITHUB_OUTPUT" + echo "osd_ref=${REQUESTED_REF:-${VAR_REF:-main}}" >> "$GITHUB_OUTPUT" + + # One leg per released engine version: run the contract queries against the + # official distribution image (which bundles the matching sql plugin) and + # export that engine's grammar bundle. + observe-released: + name: Observe engine ${{ matrix.version }} + needs: plan + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.released) }} + services: + opensearch: + image: opensearchproject/opensearch:${{ matrix.version }} + env: + discovery.type: single-node + # The lint contract only needs the PPL query and grammar endpoints, so + # run without the security plugin: no TLS or credentials to manage, and + # the observed error bodies are the engine's own rather than a proxy's. + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Wait for the engine and confirm its version + id: engine + run: | + set -euo pipefail + for i in $(seq 1 40); do + if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi + echo "waiting for engine (${i}/40)..." + sleep 5 + done + cat /tmp/root.json + reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") + echo "reported=$reported" >> "$GITHUB_OUTPUT" + # A leg mislabeled as another version would attribute drift to the wrong + # engine, so require the image to be what the matrix asked for. + case "$reported" in + ${{ matrix.version }}*) ;; + *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + esac + # The PPL plugin must actually be present, or every query would "pass" + # by failing identically. + curl -sf http://localhost:9200/_cat/plugins | grep -i sql + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # The same contract oracle the single-version workflow runs, pointed at an + # external cluster instead of a Gradle-managed one. One oracle, many + # engines: a per-version copy would be free to drift from the real check. + - name: Run contract observation against engine ${{ matrix.version }} + run: | + set -euo pipefail + mkdir -p leg + ./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha="${GITHUB_SHA}" \ + -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ + -Dppl.lint.grammar.bundle="$(pwd)/leg/ppl-grammar-bundle.json" \ + -Dppl.lint.target="$(pwd)/leg/target.json" + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-${{ matrix.version }} + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-${{ matrix.version }}-logs + path: integ-test/build/reports/** + + # Legs for engines BELOW the _grammar endpoint floor (3.6). These cannot export a + # grammar bundle, so they are observed for backend behavior only and their + # detector pass runs on the compiled-simplified surface — which is what a real + # user on such an engine gets, since no bundle can ever load there. + # + # Identical to observe-released except that `-Dppl.lint.grammar.bundle` is + # omitted: the IT skips the export when that property is unset, so no bundle + # fetch is attempted against an engine that has no such endpoint. + observe-compiled: + name: Observe engine ${{ matrix.version }} (compiled surface) + needs: plan + # An empty compiled list means "skip this surface" (the pull_request default). + if: ${{ needs.plan.outputs.compiled != '[]' }} + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + version: ${{ fromJSON(needs.plan.outputs.compiled) }} + services: + opensearch: + image: opensearchproject/opensearch:${{ matrix.version }} + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Wait for the engine and confirm its version + id: engine + run: | + set -euo pipefail + for i in $(seq 1 40); do + if curl -sf http://localhost:9200 > /tmp/root.json; then break; fi + echo "waiting for engine (${i}/40)..." + sleep 5 + done + cat /tmp/root.json + reported=$(python3 -c "import json;print(json.load(open('/tmp/root.json'))['version']['number'])") + case "$reported" in + ${{ matrix.version }}*) ;; + *) echo "::error::engine reported $reported but the matrix asked for ${{ matrix.version }}"; exit 1 ;; + esac + curl -sf http://localhost:9200/_cat/plugins | grep -i sql + + # `_cluster/health` goes GREEN before the bundled plugins finish creating + # their system indices, and the IT's first act is to wipe every non-system + # index. On 2.19 that DELETE landed while ML Commons was still initializing + # `.plugins-ml-config` and blocked until the client's 60s socket timeout, + # failing the leg before a single contract query ran. Wait for the plugin + # indices to stop appearing, so the wipe cannot race initialization. + - name: Wait for bundled plugin system indices to settle + run: | + set -euo pipefail + previous="" + stable=0 + for i in $(seq 1 30); do + current=$(curl -sf "http://localhost:9200/_cat/indices?h=index&expand_wildcards=all" \ + | sort | tr '\n' ',' || true) + if [ -n "$current" ] && [ "$current" = "$previous" ]; then + stable=$((stable + 1)) + # Three consecutive identical listings: no plugin is still creating + # indices. One match is not enough — initialization has gaps between + # an index being created and the next one starting. + if [ "$stable" -ge 3 ]; then + echo "index set stable after ${i} poll(s): $current" + exit 0 + fi + else + stable=0 + fi + previous="$current" + sleep 2 + done + # Not fatal: a slow-but-working engine should still be observed. The IT + # tolerates a wipe failure per index, and a genuinely unreachable cluster + # fails loudly in the next step anyway. + echo "::warning::plugin index set did not stabilize; continuing" + + # Probe the EXACT requests the test framework makes before any test runs. + # `OpenSearchRestTestCase.initClient` issues `GET _nodes/plugins`, and + # `wipeAllOpenSearchIndices` issues `GET _cat/indices?expand_wildcards=all`. + # A leg that dies with a bare socket timeout gives no clue which of those + # hung, so time them here where the output is readable. + - name: Probe the framework's own startup requests + run: | + set -uo pipefail + for path in "_nodes/plugins" "_cat/indices?format=json&expand_wildcards=all" "_cluster/health"; do + start=$(date +%s) + if curl -sS --max-time 30 -o /tmp/probe.out -w '%{http_code}' \ + "http://localhost:9200/${path}" > /tmp/probe.code 2>/tmp/probe.err; then + echo "OK $(($(date +%s) - start))s HTTP $(cat /tmp/probe.code) ${path} ($(wc -c < /tmp/probe.out) bytes)" + else + echo "::warning::SLOW/FAIL $(($(date +%s) - start))s ${path} $(cat /tmp/probe.err)" + fi + done + start=$(date +%s) + if curl -sS --http2 --max-time 30 -o /dev/null -w '%{http_version}' \ + "http://localhost:9200/_nodes/plugins" > /tmp/h2.out 2>/tmp/h2.err; then + echo "h2 probe: negotiated HTTP/$(cat /tmp/h2.out) in $(($(date +%s) - start))s" + else + echo "::warning::h2 probe FAILED after $(($(date +%s) - start))s: $(cat /tmp/h2.err)" + fi + # Response SIZE is the last untested difference. curl streams the body and + # does not care; the test framework calls entityAsMap on it, and + # _nodes/plugins on an engine with many bundled plugins is large. Record the + # sizes so a size-dependent hang is visible rather than inferred. + for path in "_nodes/plugins" "_nodes" "_cat/plugins"; do + bytes=$(curl -sS --max-time 30 "http://localhost:9200/${path}" | wc -c) + echo "size probe: ${path} -> ${bytes} bytes" + done + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # The curl probes above reach this engine instantly, yet the contract IT's own + # client times out on the same endpoint before any test body runs. Everything + # curl can tell us has been exhausted, so run the probe from a JVM: raw TCP, + # then HttpURLConnection, then the real OpenSearch RestClient per endpoint. + # Whichever layer stops working is the answer. + # + # `continue-on-error` because this is a diagnostic: its findings must not be + # what decides the leg, and the contract step below is still the real check. + - name: Probe REST client connectivity from a JVM + continue-on-error: true + run: | + set -uo pipefail + ./gradlew :integ-test:integTestRemote \ + --tests 'org.opensearch.sql.calcite.remote.RestClientConnectivityProbeIT' \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + --info 2>&1 | grep -E 'rest-connectivity-probe|FAILED|BUILD|tests? completed|No tests found' || true + + - name: Run contract observation against engine ${{ matrix.version }} + run: | + set -euo pipefail + mkdir -p leg + # No -Dppl.lint.grammar.bundle: this engine predates the _grammar endpoint, + # and the IT correctly exports nothing when the property is unset. + ./gradlew :integ-test:integTestRemote \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dtests.rest.cluster=localhost:9200 \ + -Dtests.cluster=localhost:9200 \ + -Dtests.clustername=docker-cluster \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha="${GITHUB_SHA}" \ + -Dppl.lint.report="$(pwd)/leg/backend-report.json" \ + -Dppl.lint.target="$(pwd)/leg/target.json" + # Mark the leg so the detect job knows to lint it on the compiled surface. + # A leg with no bundle would otherwise look like a failed export. + echo 'compiled-simplified' > leg/surface + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-${{ matrix.version }}-compiled + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-${{ matrix.version }}-compiled-logs + path: integ-test/build/reports/** + + # The PR's own engine build, so the newest point in the matrix is the code under + # review rather than the last release. Same oracle as the released legs; the only + # difference is a Gradle-managed cluster instead of a published image, which is + # why it cannot just be another matrix entry. + observe-pr-build: + name: Observe engine pr-build + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 21 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 21 + + # Observe-only here too, so this leg reports what the PR engine does rather + # than duplicating the sibling workflow's assertions. The sibling workflow + # remains the enforcing single-version check. + - name: Run contract observation against the PR build + run: | + set -euo pipefail + mkdir -p leg + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ + --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ + -Dppl.lint.report=$(pwd)/leg/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/leg/target.json" + + - name: Upload leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-pr-build + path: leg + if-no-files-found: error + + - name: Upload failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-pr-build-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** + integ-test/build/testclusters/*/logs/* + + # The PR build through the full composite/Parquet + DataFusion stack. This is + # an observation leg: route/identity/infrastructure failures are fatal, while + # backend oracles are promoted only after their captured behavior is reviewed. + observe-pr-build-analytics: + name: Observe engine pr-build (analytics) + needs: Get-CI-Image-Tag + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + ANALYTICS_FEATURE_BUILD_LATEST: https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch + container: + image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }} + options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }} + steps: + - name: Run start commands + run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }} + + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up JDK 25 + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4 + with: + distribution: 'temurin' + java-version: 25 + + - name: Resolve analytics feature build + id: analytics-build + run: | + set -euo pipefail + mkdir -p leg + requested_manifest="${ANALYTICS_FEATURE_BUILD_LATEST}/manifest.yml" + resolved_manifest=$(curl --fail --silent --show-error --location \ + --retry 3 --retry-all-errors \ + --output leg/analytics-feature-manifest.yml \ + --write-out '%{url_effective}' \ + "$requested_manifest") + artifact_root="${resolved_manifest%/manifest.yml}" + plugin_base="${artifact_root}/plugins" + native_url="${artifact_root}/dist/libopensearch_native.so" + { + echo "artifact_root=$artifact_root" + echo "plugin_base=$plugin_base" + echo "native_url=$native_url" + } >> "$GITHUB_OUTPUT" + ANALYTICS_ARTIFACT_ROOT="$artifact_root" \ + ANALYTICS_PLUGIN_BASE="$plugin_base" \ + ANALYTICS_NATIVE_URL="$native_url" \ + ANALYTICS_RESOLVED_MANIFEST="$resolved_manifest" \ + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + manifest = Path("leg/analytics-feature-manifest.yml") + context = { + "schemaVersion": 1, + "stage": "feature-build-resolved", + "sqlSha": os.environ["GITHUB_SHA"], + "executionBackend": "analytics", + "storage": "composite-parquet", + "artifactRoot": os.environ["ANALYTICS_ARTIFACT_ROOT"], + "pluginBase": os.environ["ANALYTICS_PLUGIN_BASE"], + "nativeLibraryUrl": os.environ["ANALYTICS_NATIVE_URL"], + "resolvedManifestUrl": os.environ["ANALYTICS_RESOLVED_MANIFEST"], + "manifestSha256": "sha256:" + hashlib.sha256(manifest.read_bytes()).hexdigest(), + } + Path("leg/analytics-bootstrap.json").write_text( + json.dumps(context, indent=2) + "\n", encoding="utf-8" + ) + PY + + - name: Run analytics contract observation against the PR build + id: analytics-observation + env: + ANALYTICS_FEATURE_BUILD_BASE: ${{ steps.analytics-build.outputs.plugin_base }} + ANALYTICS_NATIVE_LIB_URL: ${{ steps.analytics-build.outputs.native_url }} + run: | + set -euo pipefail + chown -R 1000:1000 "$(pwd)" + su "$(id -un 1000)" -c "./gradlew :integ-test:analyticsEnginePplLintIT \ + -PanalyticsFeatureBuildBase=${ANALYTICS_FEATURE_BUILD_BASE} \ + -PanalyticsNativeLibUrl=${ANALYTICS_NATIVE_LIB_URL} \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ + -Dppl.lint.report=$(pwd)/leg/backend-report.json \ + -Dppl.lint.grammar.bundle=$(pwd)/leg/ppl-grammar-bundle.json \ + -Dppl.lint.target=$(pwd)/leg/target.json" + + - name: Record analytics bootstrap provenance + if: ${{ always() }} + env: + OBSERVATION_OUTCOME: ${{ steps.analytics-observation.outcome }} + ANALYTICS_ARTIFACT_ROOT: ${{ steps.analytics-build.outputs.artifact_root }} + ANALYTICS_PLUGIN_BASE: ${{ steps.analytics-build.outputs.plugin_base }} + ANALYTICS_NATIVE_URL: ${{ steps.analytics-build.outputs.native_url }} + run: | + mkdir -p leg + python3 - <<'PY' + import hashlib + import json + import os + from pathlib import Path + + context_file = Path("leg/analytics-bootstrap.json") + if context_file.exists(): + context = json.loads(context_file.read_text(encoding="utf-8")) + else: + context = { + "schemaVersion": 1, + "sqlSha": os.environ["GITHUB_SHA"], + "executionBackend": "analytics", + "storage": "composite-parquet", + "artifactRoot": os.environ.get("ANALYTICS_ARTIFACT_ROOT") or None, + "pluginBase": os.environ.get("ANALYTICS_PLUGIN_BASE") or None, + "nativeLibraryUrl": os.environ.get("ANALYTICS_NATIVE_URL") or None, + } + + def describe(file): + digest = hashlib.sha256() + with file.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return { + "path": str(file), + "size": file.stat().st_size, + "sha256": "sha256:" + digest.hexdigest(), + } + + distributions = Path("integ-test/build/distributions") + native_libraries = sorted( + Path("integ-test/build/native").glob( + "*/release/libopensearch_native.so" + ) + ) + artifacts = ( + [describe(file) for file in sorted(distributions.glob("*.zip"))] + if distributions.is_dir() + else [] + ) + artifacts.extend(describe(file) for file in native_libraries) + context["stage"] = "observation-finished" + context["outcome"] = os.environ.get("OBSERVATION_OUTCOME") or "not-run" + context["effectiveJavaLibraryPaths"] = [ + str(file.parent) for file in native_libraries + ] + context["downloadedArtifacts"] = artifacts + context_file.write_text( + json.dumps(context, indent=2) + "\n", encoding="utf-8" + ) + PY + + - name: Upload analytics leg artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: ppl-lint-leg-pr-build-analytics + path: leg + if-no-files-found: error + + - name: Upload analytics failure logs + if: ${{ failure() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-leg-pr-build-analytics-logs + path: | + integ-test/build/reports/** + integ-test/build/test-results/** + integ-test/build/testclusters/*/logs/* + + # Lint each engine's exported grammar with the OSD detectors. Separate from the + # observation legs because OSD needs a newer Node/glibc than the engine image + # provides, and because one bootstrap can serve every leg. + # + # `always()` so a single broken leg still yields a report for the others: a + # partial matrix must be visibly partial, not silently absent. The aggregate + # step fails if NO leg produced a report. + detect: + name: Detect on each engine grammar + needs: + - plan + - observe-released + - observe-compiled + - observe-pr-build + - observe-pr-build-analytics + if: ${{ always() && needs.plan.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 40 + outputs: + osd_sha: ${{ steps.osd-rev.outputs.sha }} + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Download all leg artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 + with: + pattern: ppl-lint-leg-* + path: legs + + - name: Validate PR-build target identity + run: | + node scripts/ppl-lint/validate-pr-build-targets.mjs \ + --standard legs/ppl-lint-leg-pr-build/target.json \ + --analytics legs/ppl-lint-leg-pr-build-analytics/target.json \ + --standard-report legs/ppl-lint-leg-pr-build/backend-report.json \ + --analytics-report legs/ppl-lint-leg-pr-build-analytics/backend-report.json \ + --contracts integ-test/src/test/resources/ppl-lint/contracts \ + --schedule nightly + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ needs.plan.outputs.osd_repo }} + ref: ${{ needs.plan.outputs.osd_ref }} + path: .ci/OpenSearch-Dashboards + + - name: Record OSD revision + id: osd-rev + run: | + sha=$(git -C .ci/OpenSearch-Dashboards rev-parse HEAD) + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "OSD revision: \`$sha\` (${{ needs.plan.outputs.osd_repo }} @ \`${{ needs.plan.outputs.osd_ref }}\`)" >> "$GITHUB_STEP_SUMMARY" + + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + # One detector pass per leg, each against THAT engine's grammar bundle. The + # runner is the same SQL-owned script the single-version workflow uses, so + # the detector half cannot drift between the two checks. + - name: Run detectors against every engine grammar + working-directory: .ci/OpenSearch-Dashboards + run: | + set -euo pipefail + shopt -s nullglob + legs=("$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*) + if [ ${#legs[@]} -eq 0 ]; then + echo "::error::no leg artifacts were downloaded; nothing to validate." + exit 1 + fi + for leg in "${legs[@]}"; do + version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') + # A leg is compiled-surface when its observe job said so. That marker is + # what distinguishes "this engine has no _grammar endpoint" from "the + # bundle export failed", which must stay a hard error. + if [ -f "$leg/surface" ] && [ "$(cat "$leg/surface")" = 'compiled-simplified' ]; then + surface_env=(PPL_LINT_SURFACE=compiled-simplified) + echo "=== detectors vs engine $version (compiled-simplified surface) ===" + elif [ -f "$leg/ppl-grammar-bundle.json" ]; then + surface_env=(PPL_LINT_SURFACE=runtime-bundle + PPL_LINT_GRAMMAR_BUNDLE="$leg/ppl-grammar-bundle.json") + echo "=== detectors vs engine $version (runtime-bundle surface) ===" + else + # Skip the log-only artifacts an observation failure may have uploaded. + echo "skipping $leg (no grammar bundle and no compiled-surface marker)" + continue + fi + observe_env=(PPL_LINT_OBSERVE_ONLY=1) + if [ "$(jq -r '.executionBackend // empty' "$leg/target.json")" = 'analytics' ]; then + observe_env+=(PPL_LINT_OBSERVE_ANALYTICS=1) + fi + env "${surface_env[@]}" "${observe_env[@]}" \ + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_TARGET_MANIFEST="$leg/target.json" \ + PPL_LINT_REPORT="$leg/detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + > "$leg/detector.log" 2>&1 || true + # A per-leg non-zero exit is EXPECTED when that engine disagrees with + # the pinned expectation — that is the drift this workflow exists to + # report, and the aggregate step below is what classifies it. Only a + # missing report means the runner itself broke. + if [ ! -f "$leg/detector-report.json" ]; then + echo "::error::detector runner produced no report for engine $version" + tail -50 "$leg/detector.log" || true + exit 1 + fi + tail -5 "$leg/detector.log" || true + done + + # Compare every engine version against every other and against the pinned + # contracts, then print the remediation report. + - name: Aggregate drift across engine versions + id: aggregate + env: + RELEASED: ${{ needs.plan.outputs.released }} + COMPILED: ${{ needs.plan.outputs.compiled }} + run: | + set -euo pipefail + shopt -s nullglob + args=() + present=() + for leg in "$GITHUB_WORKSPACE"/legs/ppl-lint-leg-*; do + [ -f "$leg/detector-report.json" ] || continue + version=$(basename "$leg" | sed 's/^ppl-lint-leg-//') + args+=(--leg "$version=$leg") + present+=("$version") + done + if [ ${#args[@]} -eq 0 ]; then + echo "::error::no complete legs to aggregate." + exit 1 + fi + # Every version the plan asked for must have produced a leg. Aggregating + # only the survivors would report "PASS: agrees with all N versions" over + # a matrix that silently lost one — the exact vacuous pass this workflow + # exists to prevent. A dead leg is a failure, not a smaller matrix. + missing=() + # Compiled legs are labelled "-compiled" to match their artifact + # name, so they occupy their own column even when a runtime leg validated + # the same engine version. + compiled_wanted=$(echo "$COMPILED" | python3 -c " + import json,sys + print(' '.join(f'{v}-compiled' for v in json.load(sys.stdin))) + ") + for want in $(echo "$RELEASED" | python3 -c "import json,sys;print(' '.join(json.load(sys.stdin)))") $compiled_wanted pr-build pr-build-analytics; do + found=no + for have in "${present[@]}"; do + [ "$have" = "$want" ] && found=yes && break + done + [ "$found" = yes ] || missing+=("$want") + done + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::planned engine leg(s) produced no report: ${missing[*]}. Check those observe jobs; the matrix is incomplete so its result would be misleading." + exit 1 + fi + node "$GITHUB_WORKSPACE/scripts/ppl-lint/aggregate-versions.mjs" \ + --contracts "$GITHUB_WORKSPACE/integ-test/src/test/resources/ppl-lint/contracts" \ + --out "$GITHUB_WORKSPACE/drift-report.json" \ + --summary "$GITHUB_STEP_SUMMARY" \ + --all-rules \ + --observe-analytics \ + "${args[@]}" + + - name: Upload drift report + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-multiversion-drift + path: | + drift-report.json + legs/**/detector-report.json + legs/**/detector.log + legs/**/target.json + legs/**/analytics-bootstrap.json + legs/**/analytics-feature-manifest.yml + + # Discovery: harvest queries from OSD's own lint tests, run both halves over them, + # and report detector/engine disagreements as LEADS. + # + # Why this is separate from `detect`, and why it can never fail the build: + # + # The enforced corpus is hand-pinned — every expectation is a reviewed claim, which + # is what lets a mismatch red the build. That corpus is also small (about one + # trigger per rule), and `classifyRelaxationScope` needs SEVERAL triggers per rule + # to tell a FULL engine fix (version-scope the rule away) from a PARTIAL one + # (narrow the detector). Those need opposite actions, so with one trigger the + # advice can be confidently wrong. + # + # This job supplies that trigger variety from queries OSD's own detector authors + # already wrote. It pins NOTHING: roles are derived from real detector output and + # the engine supplies the other half, so no expectation is ever auto-generated. + # An auto-derived expectation could only confirm current behavior — locking in + # whatever the detector does today, bugs included. + # + # `continue-on-error` AND a zero exit from the labeler: a finding here is a lead to + # investigate, not a proven defect, and blocking unrelated PRs on an auto-generated + # guess would poison the whole check's credibility. + discovery: + name: Discovery corpus (harvested, not enforced) + # Only `plan`, for the OSD target and the engine version. Deliberately NOT the + # observe legs: discovery runs its own engine and harvests its own queries, so + # depending on them would idle this job behind ~30 minutes of matrix work it + # never reads, and a failed leg would block a report that does not need it. + needs: plan + continue-on-error: true + runs-on: ubuntu-latest + timeout-minutes: 40 + services: + opensearch: + image: opensearchproject/opensearch:${{ needs.plan.outputs.discovery_engine }} + env: + discovery.type: single-node + DISABLE_SECURITY_PLUGIN: 'true' + DISABLE_INSTALL_DEMO_CONFIG: 'true' + OPENSEARCH_JAVA_OPTS: -Xms1g -Xmx1g + ports: + - 9200:9200 + options: >- + --health-cmd "curl -sf http://localhost:9200/_cluster/health || exit 1" + --health-interval 15s + --health-timeout 10s + --health-retries 20 + --health-start-period 60s + steps: + - name: Checkout SQL pull request + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Checkout OpenSearch-Dashboards + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ needs.plan.outputs.osd_repo }} + ref: ${{ needs.plan.outputs.osd_ref }} + path: .ci/OpenSearch-Dashboards + + - name: Set up Node from OSD .nvmrc + uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4 + with: + node-version-file: .ci/OpenSearch-Dashboards/.nvmrc + + - name: Pin Yarn from OSD engines + working-directory: .ci/OpenSearch-Dashboards + run: | + yarn_range=$(node -e "process.stdout.write(require('./package.json').engines.yarn)") + yarn_version=$(echo "$yarn_range" | sed -E 's/[^0-9.]//g') + npm install -g "yarn@${yarn_version}" + + - name: Cache OSD Yarn dependencies + uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4 + with: + path: | + ~/.cache/yarn + key: ${{ runner.os }}-osd-yarn-${{ hashFiles('.ci/OpenSearch-Dashboards/yarn.lock') }} + restore-keys: | + ${{ runner.os }}-osd-yarn- + + - name: Bootstrap OpenSearch-Dashboards + working-directory: .ci/OpenSearch-Dashboards + run: | + for i in 1 2 3; do + yarn osd bootstrap && exit 0 + echo "Bootstrap attempt $i failed, retrying in 10s..." + sleep 10 + done + exit 1 + + # The rule id list comes from the OSD catalog being validated, not a hardcoded + # copy: attribution keys off `describe('')` titles, so a stale list + # would silently stop harvesting queries for any newly added rule. + - name: Harvest the discovery corpus from OSD's lint tests + run: | + set -euo pipefail + node -e " + const c = require('./.ci/OpenSearch-Dashboards/packages/osd-monaco/src/ppl/lint/rules_catalog.json'); + process.stdout.write(JSON.stringify(c.map((r) => r.id))); + " > /tmp/catalog-rules.json + node scripts/ppl-lint/harvest-queries.mjs \ + --osd .ci/OpenSearch-Dashboards \ + --catalog-rules @/tmp/catalog-rules.json \ + --index opensearch-sql_test_index_account \ + --out "$GITHUB_WORKSPACE/discovery-corpus.json" \ + --specs-out "$GITHUB_WORKSPACE/discovery-specs" + + # Seed the one index every harvested query was rewritten onto. Without it the + # engine rejects everything with IndexNotFoundException — which the labeler + # would correctly suppress as uninformative, yielding a run that reports + # nothing at all. + - name: Seed the fixture index + run: | + set -euo pipefail + for i in $(seq 1 40); do + curl -sf http://localhost:9200 > /dev/null && break + echo "waiting for engine (${i}/40)..." + sleep 5 + done + curl -sf -X PUT "http://localhost:9200/opensearch-sql_test_index_account" \ + -H 'content-type: application/json' -d '{ + "mappings": { "properties": { + "account_number": { "type": "long" }, + "balance": { "type": "long" }, + "age": { "type": "integer" }, + "status": { "type": "keyword" }, + "firstname": { "type": "text" }, + "lastname": { "type": "text" }, + "msg": { "type": "text" }, + "body": { "type": "text" }, + "raw": { "type": "object", "enabled": false } + } } + }' + curl -sf -X POST "http://localhost:9200/opensearch-sql_test_index_account/_doc?refresh=true" \ + -H 'content-type: application/json' \ + -d '{"account_number":1,"balance":39225,"age":32,"status":"ok","firstname":"Amber","lastname":"Duke","msg":"took 42ms","body":"INFO started"}' + + # Export this engine's grammar bundle so the detector pass can run on the + # RUNTIME surface. That surface matters more than the compiled one here: the + # four `runtimeOnly` rules (union/multisearch/replace arity) are SKIPPED by + # lint_runner on the compiled grammar because the productions they walk do not + # exist there — so a compiled-only discovery run cannot observe them at all, + # and three of the four ship at error severity. + - name: Export the engine grammar bundle + id: bundle + run: | + set -uo pipefail + if curl -sf --max-time 60 "http://localhost:9200/_plugins/_ppl/_grammar" \ + -o "$GITHUB_WORKSPACE/discovery-bundle.json"; then + hash=$(python3 -c " + import json + print(json.load(open('$GITHUB_WORKSPACE/discovery-bundle.json')).get('grammarHash','')) + ") + python3 -c " + import json + json.dump({'schemaVersion': 2, + 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '$hash', + 'grammarBundle': 'discovery-bundle.json', + 'executionBackend': 'standard', + 'storage': 'lucene', + 'shardCount': 1}, + open('$GITHUB_WORKSPACE/discovery-target.json','w')) + " + echo "surface=runtime-bundle" >> "$GITHUB_OUTPUT" + else + # Not fatal. Discovery is best-effort, and the compiled surface still + # covers 12 of the rules — a lead-generator that produces nothing because + # one endpoint was unavailable is worse than one with narrower coverage. + # The surface is recorded in the report, so a reader can see which ran. + echo "::warning::_grammar export failed; falling back to the compiled surface (runtimeOnly rules will not be observed)." + echo "surface=compiled-simplified" >> "$GITHUB_OUTPUT" + python3 -c " + import json + json.dump({'schemaVersion': 2, + 'engineVersion': '${{ needs.plan.outputs.discovery_engine }}', + 'grammarHash': '', + 'grammarBundle': '', + 'executionBackend': 'standard', + 'storage': 'lucene', + 'shardCount': 1}, + open('$GITHUB_WORKSPACE/discovery-target.json','w')) + " + fi + + - name: Run the detectors over the discovery corpus + working-directory: .ci/OpenSearch-Dashboards + env: + SURFACE: ${{ steps.bundle.outputs.surface }} + run: | + set -uo pipefail + # Seeded with a harmless assignment rather than left empty: under `set -u`, + # expanding an empty array as "${a[@]}" is an unbound-variable error in bash + # before 4.4, which would crash the compiled-surface fallback — the very + # path that only runs when something else already went wrong. + extra=(PPL_LINT_DISCOVERY=1 + PPL_LINT_TARGET_MANIFEST="$GITHUB_WORKSPACE/discovery-target.json") + if [ "$SURFACE" = 'runtime-bundle' ]; then + extra+=(PPL_LINT_GRAMMAR_BUNDLE="$GITHUB_WORKSPACE/discovery-bundle.json" + ) + fi + # A non-zero exit is EXPECTED and ignored: the generated specs carry + # placeholder expectations, so the runner reports a "failure" for every + # query whose real diagnostic count differs. Only the report is read. + env "${extra[@]}" \ + PPL_LINT_SURFACE="$SURFACE" \ + PPL_LINT_CONTRACT_DIR="$GITHUB_WORKSPACE/discovery-specs" \ + PPL_LINT_SCHEDULE=nightly \ + PPL_LINT_REPORT="$GITHUB_WORKSPACE/discovery-detector-report.json" \ + node -r ./src/setup_node_env \ + "$GITHUB_WORKSPACE/scripts/ppl-lint/run-frontend-contract.mjs" \ + > "$GITHUB_WORKSPACE/discovery-detector.log" 2>&1 || true + if [ ! -f "$GITHUB_WORKSPACE/discovery-detector-report.json" ]; then + echo "::warning::the detector runner produced no discovery report; skipping." + tail -50 "$GITHUB_WORKSPACE/discovery-detector.log" || true + fi + + - name: Probe the engine with the discovery corpus + run: | + set -euo pipefail + node scripts/ppl-lint/probe-discovery-backend.mjs \ + --corpus discovery-corpus.json \ + --endpoint http://localhost:9200 \ + --out discovery-backend-report.json + + - name: Label and report + run: | + set -euo pipefail + if [ ! -f discovery-detector-report.json ]; then + echo "::warning::no detector report; nothing to label." + exit 0 + fi + node scripts/ppl-lint/label-discovery.mjs \ + --corpus discovery-corpus.json \ + --detector discovery-detector-report.json \ + --backend discovery-backend-report.json \ + --version "${{ needs.plan.outputs.discovery_engine }}" \ + --out discovery-findings.json \ + --summary "$GITHUB_STEP_SUMMARY" + + - name: Upload discovery artifacts + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + continue-on-error: true + with: + name: ppl-lint-discovery + path: | + discovery-corpus.json + discovery-findings.json + discovery-detector-report.json + discovery-backend-report.json + discovery-detector.log diff --git a/.github/workflows/ppl-lint-rule-validation.yml b/.github/workflows/ppl-lint-rule-validation.yml index 543139e3ebb..078fd285494 100644 --- a/.github/workflows/ppl-lint-rule-validation.yml +++ b/.github/workflows/ppl-lint-rule-validation.yml @@ -1,5 +1,12 @@ name: PPL lint rule validation +permissions: + contents: read + +concurrency: + group: ppl-lint-rule-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + # Cross-repository check: the OpenSearch-Dashboards (OSD) PPL lint detectors and # the SQL backend must agree on the SAME candidate runtime grammar. A shared, # reviewed corpus of contract files pins each rule's OSD detector diagnostic @@ -134,6 +141,8 @@ jobs: su "$(id -un 1000)" -c "./gradlew :integ-test:integTest \ --tests org.opensearch.sql.calcite.remote.PplLintRuleValidationIT \ -Dppl.lint.schedule=${{ steps.schedule.outputs.value }} \ + -Dppl.lint.execution_backend=standard \ + -Dppl.lint.sql_sha=${GITHUB_SHA} \ -Dppl.lint.report=$(pwd)/backend-report.json \ -Dppl.lint.grammar.bundle=$(pwd)/ppl-grammar-bundle.json \ -Dppl.lint.target=$(pwd)/target.json" diff --git a/docs/dev/ppl-lint-analytics-engine-ci-validation.md b/docs/dev/ppl-lint-analytics-engine-ci-validation.md new file mode 100644 index 00000000000..3777f561dc7 --- /dev/null +++ b/docs/dev/ppl-lint-analytics-engine-ci-validation.md @@ -0,0 +1,811 @@ +# Analytics Engine Coverage for PPL Lint CI Validation + +- **Status:** Validated for phased implementation +- **Last updated:** 2026-07-28 +- **Scope:** PPL lint contract validation in + `.github/workflows/ppl-lint-rule-validation.yml` and + `.github/workflows/ppl-lint-multiversion-validation.yml` + +## 1. Summary + +The PPL lint CI contract currently compares OpenSearch Dashboards (OSD) +detectors with the standard SQL execution route only. It does not prove that +the same lint diagnostics are correct when a query is routed through the +analytics engine and executed by DataFusion over composite/Parquet storage. + +This design adds analytics-engine coverage by: + +1. Running the existing `PplLintRuleValidationIT` corpus against a dedicated, + full-stack analytics-engine test cluster. +2. Making execution backend an explicit contract and artifact dimension, + separate from OpenSearch version, Calcite applicability, and grammar + surface. +3. Failing if the analytics lane silently falls back to the standard route. +4. Running the OSD detector comparison against both standard and analytics + backend reports while bootstrapping OSD only once. +5. Shipping the lane as non-enforcing observation first, then adding it to the + stable required result after its artifacts, expectations, and reliability + meet the promotion criteria in this document. + +The initial implementation covers the SQL pull request build on one shard. It +does not add a Cartesian product of analytics backends, released OpenSearch +versions, grammar surfaces, and shard counts. + +## 2. Current State + +### 2.1 Required PPL lint validation + +`.github/workflows/ppl-lint-rule-validation.yml` is a three-job pipeline: + +```text +backend-validation + -> detector-validation + -> validation-result +``` + +- `backend-validation` runs `PplLintRuleValidationIT` against the ordinary + Gradle `integTest` cluster. That cluster installs SQL, Job Scheduler, and + Geospatial, but not the analytics-engine stack. +- The integration test executes every scheduled trigger and control query + against `POST /_plugins/_ppl`, then exports: + - `ppl-grammar-bundle.json` + - `target.json` + - `backend-report.json` +- `detector-validation` bootstraps OSD, runs its production headless PPL lint + API against the exported grammar, and compares detector output with the + backend report. +- `validation-result` uses `if: always()` and fails unless both producer jobs + succeeded. This is the stable branch-protection check. + +The multi-version companion workflow repeats the same contract against +released standard engines and the pull request build. Its current dimensions +are OpenSearch version and grammar surface. + +### 2.2 Existing analytics-engine support + +The repository already contains most of the required test infrastructure: + +- `integ-test/build.gradle` can download the analytics engine, Arrow, + composite engine, Parquet data format, and Lucene/DataFusion backend plugin + ZIPs. +- The full analytics stack is already configured for + `analyticsEngineProfileIT` and `analyticsEngineSecurityIT`. +- `-Dtests.analytics.parquet_indices=true` makes helper-created fixtures use + composite/Parquet storage. +- `SQLIntegTestCase` applies the corresponding cluster defaults before fixture + creation. +- `PPLIntegTestCase.isAnalyticsParquetIndicesEnabled()` exposes the active + route to tests. +- `integTestRemote` already forwards the analytics fixture properties. +- `CalciteAnalyticsDatetimeWireFormatIT` demonstrates route attestation using + explain output: analytics plans contain + `LogicalTableScan(table=[[opensearch,` and not + `CalciteLogicalIndexScan`. + +### 2.3 Gap in the existing analytics workflow + +`.github/workflows/analytics-engine-compat.yml` runs only +`AnalyticsEngineCompatIT`. Its purpose is plugin coexistence. Its PPL assertion +uses the `rest` row source, which is explicitly excluded from analytics +routing. The workflow can therefore pass without executing a PPL query through +DataFusion. + +The `analyticsEngineCompat` cluster is also intentionally smaller than the +stack required for real analytics execution. It does not install the composite +engine, Parquet data format, or both analytics backends. + +### 2.4 Terminology + +The following dimensions must remain independent: + +| Dimension | Examples | Meaning | +| --- | --- | --- | +| Engine version | `3.7.0`, `3.8.0-SNAPSHOT` | OpenSearch/SQL product version | +| Grammar surface | `runtime-bundle`, `compiled-simplified` | Grammar used by OSD lint | +| Lint/planner applicability | `engine: "calcite"` | Existing OSD rule applicability | +| Execution backend | `standard`, `analytics` | SQL execution route selected at runtime | +| Storage | `lucene`, `composite-parquet` | Fixture storage that drives routing | + +Analytics uses Calcite planning, so treating `analytics` as another value of +the existing `engine` field would be incorrect. Treating it as another engine +version would also cause the drift analyzer to recommend version scoping for a +backend-specific difference. + +## 3. Problem Statement + +A lint rule is presented to users before query execution. OSD currently has no +analytics-route signal in the lint context, so the same detector result applies +whether the selected index later uses the standard or analytics route. + +The current CI can miss these failures: + +1. A detector reports an error for a query that the analytics backend accepts. + This is a false positive for analytics users. +2. A detector is silent for a query rejected only by the analytics route. This + is a false negative for analytics users. +3. A control query passes on the standard route but fails on analytics. +4. An analytics test is configured incorrectly and silently executes on the + standard route, producing a vacuous green result. +5. Standard and analytics observations are stored under the same product + version, causing aggregation to overwrite or misclassify one of them. +6. A required job consumes mutable `feature-datafusion/latest` artifacts, so a + rerun can test a different stack without recording that change. + +## 4. Goals and Non-Goals + +### 4.1 Goals + +- Run every scheduled PPL lint trigger and control against the pull request's + analytics route. +- Reuse the existing contract corpus and Java integration-test oracle. +- Use byte-identical query text, the same SQL commit, the same runtime grammar, + the same OSD commit, and the same frontend lint context for both backends. +- Represent execution backend in contracts, reports, manifests, summaries, and + aggregation keys. +- Prove that the analytics plugin stack is installed, fixtures are + composite/Parquet, routing selected analytics, and DataFusion executed a + canary query. +- Distinguish backend-route divergence from version drift. +- Fail closed on missing reports, missing expectations, route fallback, + incomplete matrices, or inconsistent grammar identity. +- Produce enough artifacts to reproduce infrastructure and semantic failures. +- Keep pull request wall-clock growth bounded by running backend jobs in + parallel and bootstrapping OSD once. + +### 4.2 Non-goals + +- Replacing the existing broad analytics compatibility, security, or profile + suites. +- Running the entire PPL integration-test suite in the lint validation job. +- Adding browser, Monaco, or a running OSD server. +- Performance or benchmark validation. +- Testing every released OpenSearch version with every analytics stack in the + first release. +- Adding multi-shard analytics coverage to the required lint check. +- Automatically accepting known analytics limitations through broad Gradle + exclusions or JUnit assumptions. +- Changing production routing solely to make the test easier. + +## 5. Design Invariants + +The implementation must preserve these invariants: + +1. **Same SQL candidate:** both backend lanes build the same checked-out SQL + commit. +2. **Same grammar:** both lanes export a runtime bundle. Their engine version + and grammar hash must match before detector validation starts. +3. **Same OSD candidate:** both detector comparisons use one resolved OSD SHA + and one OSD bootstrap. +4. **Same queries:** standard, analytics, and detector passes read the same + contract files and substitute the same index names. +5. **Explicit identity:** every target and report names its execution backend. + Missing or conflicting identity is an infrastructure failure. +6. **Proven route:** setting `tests.analytics.parquet_indices=true` is not + sufficient evidence. The analytics lane must attest the installed plugins, + index settings, explain plan, and a profiled execution. +7. **No semantic retry:** downloads and cluster startup may be retried within + bounded limits. Contract queries and assertions are executed once. +8. **No vacuous pass:** missing queries, reports, detector rows, route evidence, + or planned matrix legs fail or become an explicit non-applicable result. +9. **No implicit fallback:** the analytics lane must never count a standard + route result as analytics coverage. +10. **One detector oracle:** detector count and severity remain route + independent until OSD exposes an execution-backend lint context. +11. **Complete contracts:** every selected expectation names exactly the same + query keys as the contract's top-level `queries` map. Duplicate or missing + report rows are infrastructure failures. +12. **Strict artifacts:** requested targets and reports must exist, parse, and + agree on execution identity. Writers and consumers fail rather than degrade + to an identity-free or differential-free run. + +## 6. Target Identity + +`target.json` currently records only engine version, grammar hash, and bundle +name. It will move to schema version 2 and include execution identity: + +```json +{ + "schemaVersion": 2, + "sqlSha": "...", + "engineVersion": "3.8.0-SNAPSHOT", + "grammarHash": "sha256:...", + "grammarBundle": "ppl-grammar-bundle.json", + "executionBackend": "analytics", + "storage": "composite-parquet", + "shardCount": 1, + "analyticsStack": { + "source": "immutable feature-build URL", + "buildId": "...", + "components": [ + { + "name": "analytics-engine", + "version": "3.8.0-SNAPSHOT", + "sha256": "..." + } + ] + }, + "routeAttestation": { + "pluginsVerified": true, + "clusterSettingsVerified": true, + "fixtureIndicesVerified": true, + "explainVerified": true, + "profiledExecutionVerified": true + } +} +``` + +For the standard route: + +```json +{ + "executionBackend": "standard", + "storage": "lucene", + "shardCount": 1 +} +``` + +The backend report, detector report, drift report, and run manifest will also +carry `executionBackend`. Aggregation keys become: + +```text +(leg label, engine version, grammar surface, execution backend) +``` + +The leg label remains the presentation key because multiple legs can share the +same engine version. + +## 7. Contract Schema + +### 7.1 Schema version 4 + +Detector expectations are shared, while backend oracles are keyed by execution +backend: + +```json +{ + "schemaVersion": 4, + "ruleId": "union-min-datasets", + "index": "opensearch-sql_test_index_account", + "queries": { + "union-single-dataset": { + "role": "trigger", + "query": "union [ source={{index}} ]" + }, + "union-two-datasets-control": { + "role": "control", + "query": "union [ source={{index}} ] [ source={{index}} ]" + } + }, + "grammarSurface": "runtime-bundle", + "schedule": "pr", + "backendFixture": { + "indices": ["ACCOUNT"], + "clusterSettings": { + "calcite": true, + "calciteFallback": false + } + }, + "frontendContext": { + "isCalcite": true + }, + "expectations": [ + { + "version": ">=3.7.0", + "engine": "calcite", + "queries": { + "union-single-dataset": { + "detectorCount": 1, + "severity": "error", + "backends": { + "standard": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException", + "reason": "Union command requires at least two datasets. Provided: 1" + } + } + }, + "analytics": { + "kind": "rejection", + "httpStatus": 400, + "body": { + "status": 400, + "error": { + "type": "IllegalArgumentException" + } + } + } + } + }, + "union-two-datasets-control": { + "detectorCount": 0, + "backends": { + "standard": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + }, + "analytics": { + "kind": "result-shape", + "httpStatus": 200, + "expect": { + "datarowsNonEmpty": true + } + } + } + } + } + } + ] +} +``` + +The existing `engine` field keeps its current meaning. It is not renamed to +avoid mixing this work with an unrelated contract migration. + +### 7.2 Compatibility and migration + +- A schema version 3 `backend` object is read as `backends.standard`. It is not + used as an implicit analytics oracle. +- Observation can begin before every analytics oracle is reviewed. In + observation mode, a missing analytics oracle executes the query once and + records `coverage-missing` plus the raw backend result; it does not score that + result against the standard oracle. Infrastructure and route-attestation + failures still fail the lane. +- Enforcement requires `backends.analytics` for every selected query. +- An unknown execution backend is a contract error. +- An unknown schema version is a contract error. +- More than one version/planner expectation match remains an error. +- A selected expectation must name exactly the top-level contract query set. +- The Java test and Node runner must implement identical selection behavior. + +An explicit non-applicable form is permitted only when the fixture cannot +meaningfully exercise analytics: + +```json +{ + "kind": "not-applicable", + "reason": "Fixture field type cannot be represented by composite/Parquet storage", + "owner": "@analytics-team", + "issue": "https://github.com/opensearch-project/sql/issues/..." +} +``` + +Rules in the required `defaultError` set cannot be promoted while their +analytics oracle is non-applicable. For other rules, non-applicable entries +remain visible in the report and require an owner and issue. + +### 7.3 Differential policy + +| Case | Detector requirement | Backend requirement | +| --- | --- | --- | +| Control | Zero diagnostics | Every applicable backend accepts | +| Rejection trigger | Expected diagnostic count and severity | Every applicable backend rejects with its reviewed error shape | +| Advisory trigger | Expected diagnostic count and severity | Backend matches its reviewed acceptance/result-shape oracle | +| Missing backend oracle | Not scored | Coverage failure | +| Backend transport error | Not scored | Infrastructure/inconclusive failure, never acceptance | + +If an error rule fires while analytics accepts the trigger, the result is +`execution-backend-divergence`. The remediation must not recommend changing an +OpenSearch version range. Because OSD currently lacks backend context, the +choices are to make the rule valid for both routes, narrow the detector to +behavior common to both, disable it, or first add a reliable backend signal to +the OSD lint context. + +## 8. Analytics Test Cluster and Gradle Task + +### 8.1 Chosen approach + +Add a dedicated Gradle-managed cluster and task: + +```text +testClusters.analyticsEnginePplLint +:integ-test:analyticsEnginePplLintIT +``` + +The cluster will install: + +- Job Scheduler +- Arrow Base +- Arrow Flight RPC +- Analytics Engine +- Composite Engine +- Parquet Data Format +- Analytics Backend Lucene +- Analytics Backend DataFusion +- The SQL plugin built from the current checkout + +It will reuse the native-access, Netty, and experimental feature settings used +by the existing full-stack profile/security clusters. Shared cluster +configuration should be extracted into a small Gradle helper if that can be +done without changing those tasks' behavior. + +The task will: + +- Depend on all analytics plugin downloads and SQL `bundlePlugin`. +- Filter to `PplLintRuleValidationIT`. +- Set `tests.analytics.parquet_indices=true`. +- Set `tests.analytics.num_shards=1`. +- Set `ppl.lint.execution_backend=analytics`. +- Forward the existing `ppl.lint.*` paths and schedule. +- Run as a non-root user in CI. + +Example invocation: + +```bash +./gradlew :integ-test:analyticsEnginePplLintIT \ + -Dppl.lint.execution_backend=analytics \ + -Dppl.lint.schedule=nightly \ + -Dppl.lint.observe.only=true \ + -Dppl.lint.report="$PWD/leg/backend-report.json" \ + -Dppl.lint.grammar.bundle="$PWD/leg/ppl-grammar-bundle.json" \ + -Dppl.lint.target="$PWD/leg/target.json" +``` + +The task sets the analytics fixture properties itself so a caller cannot +accidentally request an analytics report while creating Lucene fixtures. + +### 8.2 Why not the alternatives + +**Reuse `analyticsEngineCompatIT`:** rejected because its cluster lacks the full +execution stack and its test intentionally avoids analytics routing. + +**Provision an external cluster and use `integTestRemote`:** the remote task is +a valid future path for released analytics stacks, but it requires separate +cluster lifecycle, plugin installation, and SQL-plugin provenance checks. A +managed cluster is simpler and guarantees that the SQL plugin comes from the +current checkout. + +**Run the full PPL integration suite:** rejected for the required lint check. +It adds unrelated capability exclusions, runtime, and flakiness without +improving the detector contract. + +## 9. Route Attestation + +The analytics integration test will perform attestation before scoring any +contract: + +1. Query `/_cat/plugins?format=json` and require every plugin in the full stack. +2. Verify plugin versions are compatible with the OpenSearch/SQL version. +3. Read cluster settings and require composite data format defaults. +4. Read settings for every fixture index and require: + - `index.pluggable.dataformat.enabled=true` + - `index.pluggable.dataformat=composite` + - `index.composite.primary_data_format=parquet` +5. Run one valid explain canary per fixture index: + + ```text + source= | head 1 + ``` + + Require `LogicalTableScan(table=[[opensearch,` and reject + `CalciteLogicalIndexScan`. +6. Run the same canary with `profile=true` and require at least one successful + execution stage. Record every `execution_type`. Before required promotion, + pin and require the exact DataFusion-specific marker exposed by the locked + analytics stack. A generic non-empty profile is sufficient only for the + observation lane. +7. Write the attestation outcome into `target.json`. + +Invalid trigger queries may fail before DataFusion execution. Such results are +observations from an analytics-configured, route-attested environment, not +claims that DataFusion executed the invalid query. The canary proves that each +fixture is capable of analytics execution. A static +`cluster.pluggable.dataformat=composite` startup setting is also required so +query-initial parse failures see the same routing configuration as valid +queries. + +Attestation uses assertions, not JUnit assumptions. A missing plugin or legacy +explain plan fails the lane. + +## 10. CI Workflow + +### 10.1 Final required topology + +```text +Get-CI-Image-Tag + |-------------------------------| + v v +standard-backend-validation analytics-backend-validation + | | + |---- standard artifacts |---- analytics artifacts + \ / + v v + detector-validation + (one OSD checkout/bootstrap, + two backend comparisons) + | + v + validation-result +``` + +The backend jobs run in parallel. The analytics job uses JDK 25 to match the +existing analytics compatibility workflow; the standard job keeps its current +JDK. + +Artifacts use distinct names and directories: + +```text +ppl-lint-backend-standard/ +ppl-lint-backend-analytics/ +ppl-lint-backend-standard-logs/ +ppl-lint-backend-analytics-logs/ +``` + +Before linting, `detector-validation` verifies: + +- Both target manifests exist. +- Both backend reports are non-empty. +- Both targets report the expected execution backend. +- Both targets report the same engine version and grammar hash. +- Analytics route attestation is complete. +- Every requested report exists, is non-empty, contains no duplicate identities, + and agrees with its target's execution backend. + +It then invokes `run-frontend-contract.mjs` twice against the same OSD checkout +and runtime grammar: + +```text +standard backend report -> detector-standard-report.json +analytics backend report -> detector-analytics-report.json +``` + +The duplicate detector pass costs seconds; the OSD bootstrap dominates the +job. Two explicit invocations are lower risk than redesigning the runner to +accept an arbitrary report collection. The result job also compares normalized +detector rows: rule/query identity, count, severity, and any asserted message +match. Equal counts alone are not sufficient parity. + +`validation-result` continues to use `if: always()` and becomes red unless all +three validation jobs succeeded. A skipped detector caused by either backend +failure therefore cannot appear green. + +### 10.2 Multi-version workflow + +The first analytics leg is `pr-build-analytics`. It is not added to every +released version: + +| Leg | Version | Grammar surface | Execution backend | +| --- | --- | --- | --- | +| Existing released legs | Released matrix | Runtime/compiled as configured | Standard | +| `pr-build` | Pull request build | Runtime bundle | Standard | +| `pr-build-analytics` | Pull request build | Runtime bundle | Analytics | + +`aggregate-versions.mjs` must understand the backend dimension before this leg +is added. It reports backend divergence separately and never turns an +analytics-only difference into version-scoping advice. + +The discovery corpus remains standard-only in the initial implementation. It +has no reviewed oracle and should not expand the analytics rollout's cost or +diagnostic surface. + +### 10.3 Local entry point + +`scripts/ppl-lint-rule-validation.sh` will gain an opt-in analytics mode, for +example `RUN_ANALYTICS=1`. It will support the existing local ZIP override +properties. Standard local behavior remains unchanged. + +## 11. Artifact Provenance + +The current Gradle default uses a mutable +`feature-datafusion/latest/linux/x64` URL. This is acceptable for early +observation but not for a required check. + +Before promotion: + +1. Add a checked-in compatibility lock describing the immutable analytics + feature build for the current OpenSearch line. +2. Add a Gradle property such as `analyticsFeatureBuildBase` so CI can pass the + immutable base while local development can retain the current default. +3. Verify SHA-256 for every downloaded plugin ZIP before cluster startup. +4. Record the immutable source, build ID, component versions, and hashes in + `target.json`. +5. Fail if installed plugin versions do not match the locked tuple. + +If an immutable artifact source cannot be provided, the analytics lane remains +non-enforcing. + +## 12. Failure Semantics + +| Failure | Classification | CI behavior | +| --- | --- | --- | +| Plugin download or checksum failure | Infrastructure | Retry download at most three times, then fail lane | +| Cluster does not become healthy | Infrastructure | Fail and upload cluster logs/thread dump | +| Required plugin absent or wrong version | Infrastructure | Fail before contracts | +| Fixture is not composite/Parquet | Route attestation | Fail before contracts | +| Explain/profile canary uses standard route | Route attestation | Fail before contracts | +| Standard and analytics grammar hashes differ | Candidate identity | Fail detector job | +| Missing/empty backend or detector report | Incomplete run | Fail; never aggregate survivors only | +| Missing analytics expectation | Coverage hole | Fail once analytics enforcement is enabled | +| Contract query transport timeout | Inconclusive run | Fail; never treat as backend acceptance | +| Trigger/control behavior differs from oracle | Semantic drift | Report backend, query, observed status/type, and remediation | +| Standard and analytics behavior differ | Execution-backend divergence | Report separately; do not suggest version scoping | +| Detector output differs between backend passes | Harness/context defect | Fail detector job | + +Semantic assertions are never retried. A retry could hide a nondeterministic +backend or detector defect. + +## 13. Diagnostics and Resource Bounds + +The analytics job will use: + +- A 30-minute GitHub job timeout. +- A bounded OpenSearch heap consistent with current workflows. +- One Netty direct arena and the existing native-access flags. +- One primary shard for required contract coverage. +- No credentials or fork secrets. +- `permissions: contents: read`. + +Always upload on failure: + +- `target.json` and analytics stack identity. +- Backend and detector reports. +- JUnit XML and HTML reports. +- Gradle test reports. +- Installed plugin list. +- Effective cluster and fixture index settings. +- Fixture mapping hashes and any fields stripped by the analytics fixture + helper. +- OpenSearch and test-cluster logs. +- Detector logs. +- Thread dumps for startup or query timeout. + +Reports must distinguish `accepted`, `rejected`, `error`, and +`not-applicable`. An absent `rejected` field is not equivalent to acceptance. + +## 14. Test Plan + +### 14.1 Harness unit tests + +Add Node tests for: + +- Schema version 3 compatibility and schema version 4 backend selection. +- Unknown or missing execution backend. +- Missing analytics oracle. +- Duplicate target identities. +- Same version with standard and analytics legs. +- Standard/analytics grammar mismatch. +- Backend transport error not being read as acceptance. +- Analytics divergence producing backend remediation, not version scoping. +- Detector parity between standard and analytics passes. +- Non-applicable handling and required-rule coverage holes. +- Summary and annotation output naming the execution backend. + +### 14.2 Java integration coverage + +Verify: + +- The standard `PplLintRuleValidationIT` behavior is unchanged. +- The analytics task installs the full stack. +- ACCOUNT and FLAT_OBJECT fixtures are composite/Parquet or fail explicitly. +- Explain and profile canaries attest the analytics route. +- Every scheduled contract emits one backend result per expected query. +- Report entries include `executionBackend`. +- A forced missing-plugin or standard-route configuration fails attestation. + +### 14.3 Workflow validation + +Use `workflow_dispatch` to validate: + +- Canonical OSD `main`. +- An explicit OSD branch/SHA. +- A successful dual-backend run. +- An intentionally wrong analytics oracle. +- An intentionally missing analytics artifact. +- A backend failure that skips detector work but still makes the final result + red. + +No production branch-protection change is made during this validation. + +## 15. Rollout + +### Phase 1: Identity and observation + +- Add execution-backend identity to targets and reports. +- Add the schema version 4 reader with version 3 compatibility. +- Make artifact consumers fail closed on missing, malformed, duplicate, or + conflicting identities. +- Add backend-aware aggregation and divergence remediation before introducing + an analytics leg. +- Add the managed analytics Gradle task and route attestation. +- Add `pr-build-analytics` to the non-required multi-version workflow. +- Missing analytics oracles are recorded as unscored coverage gaps during + observation. Infrastructure, identity, completeness, and attestation failures + remain red. Do not use `continue-on-error` inside the producer lane. + +### Phase 2: Baseline and review + +- Capture real analytics observations for the full contract corpus. +- Add reviewed analytics oracles. +- Resolve every default-error non-applicable case. +- Pin immutable analytics artifacts and verify their checksums. +- Pin the DataFusion-specific profile execution marker. +- Measure runtime and infrastructure reliability. + +Promotion requires: + +- Every scheduled contract has a reviewed analytics oracle. +- No `defaultError` contract is non-applicable. +- No unexplained semantic divergence remains. +- At least 25 consecutive green observation runs. +- At least 50 total runs with less than 1% infrastructure failure. +- Analytics job p95 runtime is at most 15 minutes. +- Artifact provenance is immutable and recorded. + +### Phase 3: Required check + +- Add `analytics-backend-validation` to the required single-version workflow. +- Make detector validation require both backend artifacts. +- Make `validation-result` require standard backend, analytics backend, and + detector success. +- Update the run manifest and PR summary to show both routes. + +There is no silent repository-variable bypass after promotion. An emergency +rollback requires an explicit workflow/branch-protection change and a tracking +issue. + +### Phase 4: Optional expansion + +After the required lane is stable, evaluate: + +- Matching released analytics stacks. +- A scheduled three-shard analytics leg. +- Analytics execution for the discovery corpus. +- Consolidating or retiring redundant parts of + `analytics-engine-compat.yml`. + +These are separate changes and are not prerequisites for initial enforcement. + +## 16. Planned File Changes + +| File | Change | +| --- | --- | +| `integ-test/build.gradle` | Add the full-stack analytics lint cluster/task and artifact lock inputs | +| `PplLintRuleValidationIT.java` | Select backend-specific oracles, attest route, and emit backend identity | +| `integ-test/src/test/resources/ppl-lint/contracts/*.spec.json` | Migrate to schema version 4 and add analytics oracles | +| `integ-test/src/test/resources/ppl-lint/contracts/manifest.json` | Bump schema metadata and document analytics coverage | +| `scripts/ppl-lint/run-frontend-contract.mjs` | Select the active backend oracle and emit backend identity | +| `scripts/ppl-lint/contract-schema.mjs` | Share strict Node schema, identity, and backend-oracle selection | +| `scripts/ppl-lint/aggregate-versions.mjs` | Key and render legs by execution backend | +| `scripts/ppl-lint/drift.mjs` | Add execution-backend divergence and remediation | +| `scripts/ppl-lint/annotate.mjs` | Attach backend-specific findings to contract declarations | +| `scripts/ppl-lint/assemble-run-manifest.mjs` | Record both targets and job results | +| `scripts/ppl-lint/__tests__/*` | Cover schema, identity, aggregation, and remediation changes | +| `.github/workflows/ppl-lint-multiversion-validation.yml` | Add the observation leg | +| `.github/workflows/ppl-lint-rule-validation.yml` | Add the required lane after promotion | +| `scripts/ppl-lint-rule-validation.sh` | Add opt-in local analytics reproduction | +| `scripts/ppl-lint/README.md` | Document backend-aware contracts and commands | +| Analytics compatibility lock (path TBD) | Pin immutable plugin URLs, versions, and SHA-256 values before required promotion | + +## 17. Success Criteria + +The work is complete when: + +1. A pull request can produce standard and analytics observations from the same + SQL commit and grammar. +2. CI proves the analytics route instead of relying on a configuration flag. +3. Every scheduled contract has an explicit analytics result. +4. Reports cannot confuse backend divergence with version drift. +5. Missing analytics coverage cannot pass as agreement. +6. The required result fails when either backend or the OSD detector contract + fails. +7. A failed run includes enough immutable identity and logs to reproduce the + target that was tested. + +## 18. Open Questions + +1. Which system owns publishing and retaining immutable analytics feature-build + tuples for required CI? +2. Should the artifact compatibility lock live in this repository or be + generated by the OpenSearch feature-build pipeline? +3. Which current contract queries produce intentional analytics behavior + differences once the first observation run is available? +4. Will OSD eventually expose a reliable execution-backend signal to lint + context? If so, detector expectations may later become backend-aware. +5. After the full semantic lane is required, does the smaller coexistence smoke + workflow still provide enough independent value to keep? diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 1d8f3af45f4..fcceb74a107 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -169,18 +169,25 @@ tasks.withType(licenseHeaders.class) { } // Forward the PPL lint rule validation contract knobs to every integ test JVM -// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), an -// optional path to write the observed backend report, and — while the cluster is -// alive — optional paths to export the candidate runtime grammar bundle and its -// target manifest for the detector-validation job. Applied globally so every +// (PplLintRuleValidationIT reads them): which schedule to run (pr|nightly), +// whether to observe rather than assert (the multi-version matrix), an optional +// path to write the observed backend report, and — while the cluster is alive — +// optional paths to export the candidate runtime grammar bundle and its target +// manifest for the detector-validation job. Applied globally so every // RestIntegTestTask that runs the class picks it up without per-task edits. +// +// Forwarded by PREFIX rather than by an explicit list. A hand-maintained list is +// a silent trap: a property the IT reads but the list omits simply never reaches +// the test JVM, with no error anywhere. That already cost one CI run — +// `ppl.lint.observe.only` was added to the IT and the workflow but not to the +// list, so a multi-version leg asserted expectations pinned for a DIFFERENT +// engine version and failed instead of observing. Forwarding every `ppl.lint.*` +// property the invoker set means adding a knob to the IT is enough. tasks.withType(Test).configureEach { systemProperty "ppl.lint.schedule", System.getProperty("ppl.lint.schedule", "pr") - ["ppl.lint.report", "ppl.lint.grammar.bundle", "ppl.lint.target"].each { prop -> - if (System.getProperty(prop) != null) { - systemProperty prop, System.getProperty(prop) - } - } + System.properties.stringPropertyNames() + .findAll { it.startsWith("ppl.lint.") && it != "ppl.lint.schedule" } + .each { prop -> systemProperty prop, System.getProperty(prop) } } validateNebulaPom.enabled = false @@ -287,9 +294,13 @@ def getGeoSpatialPlugin() { } } -// fetch from the feature-build artifact for now (linux/x64 only; for local dev pass -PanalyticsEngineZip=/path instead). +// Fetch from the mutable feature-build artifact for observation (linux/x64 only). CI can +// select a specific build with -PanalyticsFeatureBuildBase, and local development can pass +// individual plugin ZIP overrides such as -PanalyticsEngineZip=/path. ext.pluginVersion = opensearch_version.tokenize('-')[0] -ext.featureBuildBase = "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildBase = project.findProperty('analyticsFeatureBuildBase') ?: + "https://ci.opensearch.org/ci/dbc/feature-build-opensearch/feature-datafusion/latest/linux/x64/tar/builds/opensearch/plugins" +ext.featureBuildArtifactRoot = featureBuildBase.replaceFirst('/plugins/?$', '') ext.analyticsEngineZipDest = "${buildDir}/distributions/analytics-engine-${pluginVersion}-SNAPSHOT.zip" ext.arrowFlightRpcZipDest = "${buildDir}/distributions/arrow-flight-rpc-${pluginVersion}-SNAPSHOT.zip" ext.arrowBaseZipDest = "${buildDir}/distributions/arrow-base-${pluginVersion}-SNAPSHOT.zip" @@ -298,6 +309,16 @@ ext.analyticsBackendLuceneZipDest = "${buildDir}/distributions/analytics-backend ext.parquetDataFormatZipDest = "${buildDir}/distributions/parquet-data-format-${pluginVersion}-SNAPSHOT.zip" ext.compositeEngineZipDest = "${buildDir}/distributions/composite-engine-${pluginVersion}-SNAPSHOT.zip" ext.analyticsBackendDatafusionZipDest = "${buildDir}/distributions/analytics-backend-datafusion-${pluginVersion}-SNAPSHOT.zip" +ext.analyticsNativeLibUrl = project.findProperty('analyticsNativeLibUrl') ?: + "${featureBuildArtifactRoot}/dist/libopensearch_native.so" +ext.analyticsNativeLibDest = "${buildDir}/native/${pluginVersion}/release/libopensearch_native.so" +ext.analyticsNativeLibDir = project.findProperty('nativeLibPath') ? + rootProject.file(project.findProperty('nativeLibPath')).canonicalFile : + file(analyticsNativeLibDest).parentFile.canonicalFile +ext.analyticsJavaLibraryPath = [ + analyticsNativeLibDir.absolutePath, + System.getProperty('java.library.path') +].findAll { it != null && !it.isEmpty() }.join(File.pathSeparator) task downloadAnalyticsEngineZip(type: Download) { src "${featureBuildBase}/1-analytics-engine-${pluginVersion}.zip" @@ -332,7 +353,7 @@ task downloadTestPplFrontendZip(type: Download) { } task downloadAnalyticsBackendLuceneZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-lucene-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-lucene-${pluginVersion}.zip" dest analyticsBackendLuceneZipDest overwrite false onlyIfModified true @@ -340,7 +361,7 @@ task downloadAnalyticsBackendLuceneZip(type: Download) { } task downloadParquetDataFormatZip(type: Download) { - src "${featureBuildBase}/1-parquet-data-format-${pluginVersion}.zip" + src "${featureBuildBase}/parquet-data-format-${pluginVersion}.zip" dest parquetDataFormatZipDest overwrite false onlyIfModified true @@ -348,7 +369,7 @@ task downloadParquetDataFormatZip(type: Download) { } task downloadCompositeEngineZip(type: Download) { - src "${featureBuildBase}/1-composite-engine-${pluginVersion}.zip" + src "${featureBuildBase}/2-composite-engine-${pluginVersion}.zip" dest compositeEngineZipDest overwrite false onlyIfModified true @@ -356,13 +377,57 @@ task downloadCompositeEngineZip(type: Download) { } task downloadAnalyticsBackendDatafusionZip(type: Download) { - src "${featureBuildBase}/1-analytics-backend-datafusion-${pluginVersion}.zip" + src "${featureBuildBase}/analytics-backend-datafusion-${pluginVersion}.zip" dest analyticsBackendDatafusionZipDest overwrite false onlyIfModified true onlyIf { !project.findProperty('analyticsBackendDatafusionZip') } } +task downloadAnalyticsNativeLib(type: Download) { + src analyticsNativeLibUrl + dest analyticsNativeLibDest + // The mutable observation URL can publish another build under the same + // product version. Revalidate an existing file and never expose a partial + // download to the test cluster. + overwrite true + onlyIfModified true + tempAndMove true + retries 3 + onlyIf { !project.findProperty('nativeLibPath') } + doFirst { + def osName = System.getProperty('os.name', '').toLowerCase(Locale.ROOT) + def osArch = System.getProperty('os.arch', '').toLowerCase(Locale.ROOT) + if (!osName.contains('linux') || !(osArch in ['amd64', 'x86_64'])) { + throw new GradleException( + "The default analytics native artifact is Linux/x64 only " + + "(detected ${osName}/${osArch}); pass -PnativeLibPath=.") + } + } +} + +task validateAnalyticsNativeLib { + dependsOn downloadAnalyticsNativeLib + doLast { + File nativeLib = new File(analyticsNativeLibDir, 'libopensearch_native.so') + if (!nativeLib.isFile() || !nativeLib.canRead() || nativeLib.length() == 0) { + throw new GradleException( + "Expected a readable non-empty native library at ${nativeLib}. " + + "Pass -PnativeLibPath= " + + "or -PanalyticsNativeLibUrl=.") + } + byte[] magic = new byte[4] + int bytesRead + nativeLib.withInputStream { stream -> bytesRead = stream.read(magic) } + if (bytesRead != magic.length || + (magic[0] & 0xff) != 0x7f || (magic[1] & 0xff) != 0x45 || + (magic[2] & 0xff) != 0x4c || (magic[3] & 0xff) != 0x46) { + throw new GradleException( + "Analytics native library ${nativeLib} is not an ELF shared object.") + } + } +} + def getAnalyticsEnginePlugin() { provider { (RegularFile) (() -> file(project.findProperty('analyticsEngineZip') ?: analyticsEngineZipDest)) } } @@ -445,6 +510,34 @@ testClusters { // Composite-default cluster: PPL queries route to the analytics engine unless excluded. setting 'cluster.pluggable.dataformat', 'composite' } + analyticsEnginePplLintIT { + testDistribution = 'archive' + plugin(getJobSchedulerPlugin()) + plugin(getArrowBasePlugin()) + plugin(getArrowFlightRpcPlugin()) + plugin(getAnalyticsEnginePlugin()) + plugin(getCompositeEnginePlugin()) + plugin(getParquetDataFormatPlugin()) + plugin(getAnalyticsBackendLucenePlugin()) + plugin(getAnalyticsBackendDatafusionPlugin()) + plugin ":opensearch-sql-plugin" + setting 'cluster.pluggable.dataformat.enabled', 'true' + setting 'cluster.pluggable.dataformat', 'composite' + setting 'cluster.composite.primary_data_format', 'parquet' + setting 'cluster.composite.secondary_data_formats', '[lucene]' + // Arrow Flight / streaming transport requirements + jvmArgs '--add-opens=java.base/java.nio=ALL-UNNAMED' + jvmArgs '--enable-native-access=ALL-UNNAMED' + systemProperty 'io.netty.allocator.numDirectArenas', '1' + systemProperty 'io.netty.noUnsafe', 'false' + systemProperty 'io.netty.tryUnsafe', 'true' + systemProperty 'io.netty.tryReflectionSetAccessible', 'true' + systemProperty 'opensearch.experimental.feature.pluggable.dataformat.enabled', 'true' + systemProperty 'opensearch.experimental.feature.transport.stream.enabled', 'true' + // DataFusion/parquet loads libopensearch_native.so at cluster startup. Use the + // matching feature-build artifact unless a local release directory is supplied. + systemProperty 'java.library.path', analyticsJavaLibraryPath + } } def isPrometheusRunning() { @@ -504,6 +597,26 @@ task analyticsEngineCompatIT(type: RestIntegTestTask) { } } +task analyticsEnginePplLintIT(type: RestIntegTestTask) { + useCluster testClusters.analyticsEnginePplLintIT + dependsOn downloadArrowBaseZip, downloadArrowFlightRpcZip, downloadAnalyticsEngineZip, + downloadCompositeEngineZip, downloadParquetDataFormatZip, + downloadAnalyticsBackendLuceneZip, downloadAnalyticsBackendDatafusionZip, + validateAnalyticsNativeLib + dependsOn ':opensearch-sql-plugin:bundlePlugin' + + systemProperty 'tests.analytics.parquet_indices', 'true' + systemProperty 'tests.analytics.num_shards', '1' + systemProperty 'ppl.lint.execution_backend', 'analytics' + systemProperty 'ppl.lint.analytics.stack.source', featureBuildBase + systemProperty 'tests.security.manager', 'false' + systemProperty 'project.root', project.projectDir.absolutePath + + filter { + includeTestsMatching 'org.opensearch.sql.calcite.remote.PplLintRuleValidationIT' + } +} + task analyticsEngineSecurityIT(type: RestIntegTestTask) { dependsOn downloadAnalyticsEngineZip, downloadArrowFlightRpcZip, downloadArrowBaseZip, downloadAnalyticsBackendLuceneZip, downloadParquetDataFormatZip, downloadCompositeEngineZip, downloadAnalyticsBackendDatafusionZip dependsOn ':opensearch-sql-plugin:bundlePlugin' diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java index 4185511ab04..e8dbafc42a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/PplLintRuleValidationIT.java @@ -12,7 +12,10 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; import java.util.Locale; @@ -29,14 +32,14 @@ import org.opensearch.sql.ppl.PPLIntegTestCase; /** - * Backend half of the schema-v3 PPL lint rule validation contract. + * Backend half of the schema-v3/schema-v4 PPL lint rule validation contract. * *

This test drives the live {@code POST /_plugins/_ppl} endpoint on the SQL plugin built from * the current checkout. For every contract (see {@code * src/test/resources/ppl-lint/contracts/*.spec.json}) it selects the single {@code expectations[]} * entry that matches the candidate backend version (exactly one must match, or the contract fails - * before any query runs), applies the contract's cluster settings, and asserts, per query's {@code - * backend.kind}: + * before any query runs), applies the contract's cluster settings, and asserts the oracle selected + * for {@code ppl.lint.execution_backend}: * *