diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile index 5018b4f4a318..8c91c51cf2d9 100644 --- a/ingestion/Dockerfile +++ b/ingestion/Dockerfile @@ -107,36 +107,12 @@ RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9" RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.2.2/constraints-3.10.txt" RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}" -# Patch vulnerable Hive-metastore jars bundled inside PySpark 3.5.6 (the Deltalake -# connector calls .enableHiveSupport()). SHA256-pinned to Maven Central. -# zookeeper 3.6.3 -> 3.7.2 CVE-2023-44981 (SASL quorum auth bypass); ZK 3.7 runs on Java 8+ -# jackson-mapper/core-asl 1.9.13 CVE-2019-10202 (deserialization RCE) -- removed, no fix upstream -# Derby (CVE-2022-46337) is deliberately left at the bundled 10.14.2.0. The only fixed -# release on Maven Central is 10.17.1.0, which requires Java 21 — this image ships Java 17 -# (default-jre-headless), and the Java-8/11/17 backport jars (10.14.3.0 / 10.15.2.1 / -# 10.16.1.2) were never published upstream, so no drop-in Java-17 fix exists. The CVE is -# an LDAP-authenticator injection; OM's embedded Derby metastore uses no LDAP auth, so the -# vulnerable path is unreachable. Revisit if PySpark bumps Derby or the image moves to Java 21. -# Skips cleanly when pyspark is absent (e.g. an INGESTION_DEPENDENCY build without -# the deltalake/pyspark deps); only patches jars when the pyspark jars dir exists. -RUN JARS_DIR="$(python -c 'import os,pyspark;print(os.path.join(os.path.dirname(pyspark.__file__),"jars"))' 2>/dev/null || true)" \ - && if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then \ - echo "pyspark not installed; skipping Hive-metastore jar patch"; \ - else \ - fetch_jar() { \ - wget -q "https://repo1.maven.org/maven2/$2" -O "$1" \ - && echo "$3 $1" | sha256sum -c - || exit 1; \ - }; \ - cd "${JARS_DIR}" \ - && rm -f zookeeper-*.jar zookeeper-jute-*.jar \ - jackson-mapper-asl-*.jar jackson-core-asl-*.jar \ - && fetch_jar zookeeper-3.7.2.jar \ - org/apache/zookeeper/zookeeper/3.7.2/zookeeper-3.7.2.jar \ - b12d6fb4afd7b3849d3a9a5a38b9260c23a12e1ea58ca8c8d775880249cb8eac \ - && fetch_jar zookeeper-jute-3.7.2.jar \ - org/apache/zookeeper/zookeeper-jute/3.7.2/zookeeper-jute-3.7.2.jar \ - ad15d812b1f01f373638443adcda0e23fda549d65ced2be8c4f64bef33b5d774; \ - fi +# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture. +# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after the +# final pip install; fails the build on a broken pyspark install rather than shipping +# unpatched jars. +COPY --chown=airflow:0 ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh +RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593 RUN [ $(uname -m) = "x86_64" ] \ diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci index 42f79c106d86..6092cbd1e835 100644 --- a/ingestion/Dockerfile.ci +++ b/ingestion/Dockerfile.ci @@ -144,36 +144,12 @@ RUN [ $(uname -m) = "x86_64" ] \ && pip install ".[db2]" \ || echo "DB2 not supported on ARM architectures." -# Patch vulnerable Hive-metastore jars bundled inside PySpark 3.5.6 (the Deltalake -# connector calls .enableHiveSupport()). SHA256-pinned to Maven Central. -# zookeeper 3.6.3 -> 3.7.2 CVE-2023-44981 (SASL quorum auth bypass); ZK 3.7 runs on Java 8+ -# jackson-mapper/core-asl 1.9.13 CVE-2019-10202 (deserialization RCE) -- removed, no fix upstream -# Derby (CVE-2022-46337) is deliberately left at the bundled 10.14.2.0. The only fixed -# release on Maven Central is 10.17.1.0, which requires Java 21 — this image ships Java 17 -# (default-jre-headless), and the Java-8/11/17 backport jars (10.14.3.0 / 10.15.2.1 / -# 10.16.1.2) were never published upstream, so no drop-in Java-17 fix exists. The CVE is -# an LDAP-authenticator injection; OM's embedded Derby metastore uses no LDAP auth, so the -# vulnerable path is unreachable. Revisit if PySpark bumps Derby or the image moves to Java 21. -# Skips cleanly when pyspark is absent (e.g. an INGESTION_DEPENDENCY build without -# the deltalake/pyspark deps); only patches jars when the pyspark jars dir exists. -RUN JARS_DIR="$(python -c 'import os,pyspark;print(os.path.join(os.path.dirname(pyspark.__file__),"jars"))' 2>/dev/null || true)" \ - && if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then \ - echo "pyspark not installed; skipping Hive-metastore jar patch"; \ - else \ - fetch_jar() { \ - wget -q "https://repo1.maven.org/maven2/$2" -O "$1" \ - && echo "$3 $1" | sha256sum -c - || exit 1; \ - }; \ - cd "${JARS_DIR}" \ - && rm -f zookeeper-*.jar zookeeper-jute-*.jar \ - jackson-mapper-asl-*.jar jackson-core-asl-*.jar \ - && fetch_jar zookeeper-3.7.2.jar \ - org/apache/zookeeper/zookeeper/3.7.2/zookeeper-3.7.2.jar \ - b12d6fb4afd7b3849d3a9a5a38b9260c23a12e1ea58ca8c8d775880249cb8eac \ - && fetch_jar zookeeper-jute-3.7.2.jar \ - org/apache/zookeeper/zookeeper-jute/3.7.2/zookeeper-jute-3.7.2.jar \ - ad15d812b1f01f373638443adcda0e23fda549d65ced2be8c4f64bef33b5d774; \ - fi +# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture. +# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after the +# final pip install; fails the build on a broken pyspark install rather than shipping +# unpatched jars. +COPY --chown=airflow:0 ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh +RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh # bump python-daemon for https://github.com/apache/airflow/pull/29916 RUN pip install "python-daemon>=3.0.0" diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile index c79a5b8a0963..e447dad3bd53 100644 --- a/ingestion/operators/docker/Dockerfile +++ b/ingestion/operators/docker/Dockerfile @@ -137,6 +137,13 @@ RUN [ $(uname -m) = "x86_64" ] \ && pip install "openmetadata-ingestion[db2]~=${RI_VERSION}" \ || echo "DB2 not supported on ARM architectures." +# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture. +# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after all +# pip installs so the pyspark/spacy trees exist; the script fails the build on a broken +# pyspark install rather than silently shipping unpatched jars. +COPY --chown=openmetadata:openmetadata ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh +RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh + # Uninstalling psycopg2-binary and installing psycopg2 instead # because the psycopg2-binary generates a architecture specific error # while authenticating connection with the airflow, psycopg2 solves this error diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci index 01e0f02695a1..76c81bf903e4 100644 --- a/ingestion/operators/docker/Dockerfile.ci +++ b/ingestion/operators/docker/Dockerfile.ci @@ -142,6 +142,13 @@ RUN [ $(uname -m) = "x86_64" ] \ && pip install ".[db2]" \ || echo "DB2 not supported on ARM architectures." +# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture. +# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after all +# pip installs so the pyspark/spacy trees exist; the script fails the build on a broken +# pyspark install rather than silently shipping unpatched jars. +COPY --chown=openmetadata:openmetadata ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh +RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh + # Required for Airflow DockerOperator, as we need to run the workflows from a `python main.py` command in the container. COPY ingestion/operators/docker/*.py . diff --git a/ingestion/scripts/patch_pyspark_jars.sh b/ingestion/scripts/patch_pyspark_jars.sh new file mode 100755 index 000000000000..4f7e8dec5c98 --- /dev/null +++ b/ingestion/scripts/patch_pyspark_jars.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# Copyright 2021 Collate +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Patch vulnerable jars bundled inside PySpark 3.5.6 (the Deltalake connector calls +# .enableHiveSupport()). All replacement jars are SHA256-pinned to Maven Central. +# +# zookeeper 3.6.3 -> 3.7.2 CVE-2023-44981 (SASL quorum auth bypass); ZK 3.7 runs on Java 8+ +# jackson-mapper/core-asl 1.9.13 CVE-2019-10202 (deserialization RCE) -- removed, no fix upstream +# netty-codec-http 4.1.96 -> 4.1.135 CVE-2026-42581 / CVE-2026-42584 (HTTP request smuggling); +# both fixed in 4.1.133.Final, so staying on the 4.1.x line keeps +# binary compatibility with PySpark's sibling netty 4.1.96 jars. +# +# Derby (CVE-2022-46337) is deliberately left at the bundled 10.14.2.0. The only fixed +# release on Maven Central is 10.17.1.0, which requires Java 21 — these images ship Java 17 +# (default-jre-headless), and the Java-8/11/17 backport jars (10.14.3.0 / 10.15.2.1 / +# 10.16.1.2) were never published upstream, so no drop-in Java-17 fix exists. The CVE is an +# LDAP-authenticator injection; OM's embedded Derby metastore uses no LDAP auth, so the +# vulnerable path is unreachable. Revisit if PySpark bumps Derby or the images move to Java 21. +# +# jetty-http (CVE-2026-2332) is shaded inside PySpark's hadoop-client-runtime / spark-core +# uber-jars, so it cannot be swapped as a standalone jar without repackaging Spark. Left as a +# documented residual. +# +# Skips cleanly when pyspark is genuinely absent (e.g. an INGESTION_DEPENDENCY build +# without the deltalake/pyspark deps). A pyspark that is installed but fails to import is +# treated as an error and fails the build, so a broken install can never ship unpatched +# jars silently. + +set -euo pipefail + +# Locate PySpark's bundled jars directory, distinguishing three cases via exit code: +# 0 -> installed and importable (prints the jars dir on stdout) +# 2 -> genuinely not installed (top-level pyspark module missing) -> skip +# 3 -> installed but import failed for any other reason -> fail the build +locate_pyspark_jars() { + python - <<'PY' +import importlib.util, os, sys + +# Distinguish "pyspark genuinely not installed" from "pyspark present but broken". +# find_spec only inspects the top-level pyspark package without importing it, so a +# missing pyspark.* submodule during a real import is never mistaken for absence. +try: + spec = importlib.util.find_spec("pyspark") +except Exception as exc: + sys.stderr.write(f"pyspark is installed but failed to import: {exc!r}\n") + sys.exit(3) + +if spec is None: + sys.exit(2) + +try: + import pyspark +except Exception as exc: + sys.stderr.write(f"pyspark is installed but failed to import: {exc!r}\n") + sys.exit(3) + +print(os.path.join(os.path.dirname(pyspark.__file__), "jars")) +PY +} + +# Strip spacy's bundled CI test fixture. spacy/tests/package/requirements.txt pins an old +# black, which image scanners misreport as an installed package (CVE-2026-31900). The file +# is test-only and never imported at runtime. spaCy (from the pii-processor / sample-data +# extras) installs independently of pyspark (from deltalake), so this must run on every +# path — including builds that ship spaCy but not pyspark. +# +# Deterministic, not best-effort: locate spacy via find_spec (without importing it, so an +# installed-but-broken spacy is still found), delete the fixture, then assert it is gone. +# The image scanner reads the file on disk regardless of whether spacy imports, so a broken +# spacy must not let the fixture survive while the build succeeds. +strip_spacy_scanner_fixture() { + local spacy_dir + spacy_dir="$(python - <<'PY' +import importlib.util, os, sys +spec = importlib.util.find_spec("spacy") +if spec is None or not spec.submodule_search_locations: + sys.exit(0) +print(spec.submodule_search_locations[0]) +PY +)" + if [ -z "${spacy_dir}" ] || [ ! -d "${spacy_dir}/tests" ]; then + return 0 + fi + find "${spacy_dir}/tests" -name 'requirements.txt' -delete + local leftover + leftover="$(find "${spacy_dir}/tests" -name 'requirements.txt')" + if [ -n "${leftover}" ]; then + echo "ERROR: spaCy scanner fixture still present after delete:" >&2 + echo "${leftover}" >&2 + exit 1 + fi +} + +strip_spacy_scanner_fixture + +JARS_DIR="$(locate_pyspark_jars)" && rc=0 || rc=$? + +if [ "${rc}" -eq 2 ]; then + echo "pyspark not installed; skipping PySpark jar patch" + exit 0 +elif [ "${rc}" -ne 0 ]; then + echo "ERROR: pyspark is installed but not importable; refusing to ship unpatched jars" >&2 + exit 1 +fi + +if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then + echo "ERROR: pyspark imported but its jars dir '${JARS_DIR}' is missing" >&2 + exit 1 +fi + +fetch_jar() { + wget -q "https://repo1.maven.org/maven2/$2" -O "$1" + echo "$3 $1" | sha256sum -c - +} + +cd "${JARS_DIR}" + +rm -f zookeeper-*.jar zookeeper-jute-*.jar \ + jackson-mapper-asl-*.jar jackson-core-asl-*.jar \ + netty-codec-http-*.jar + +fetch_jar zookeeper-3.7.2.jar \ + org/apache/zookeeper/zookeeper/3.7.2/zookeeper-3.7.2.jar \ + b12d6fb4afd7b3849d3a9a5a38b9260c23a12e1ea58ca8c8d775880249cb8eac + +fetch_jar zookeeper-jute-3.7.2.jar \ + org/apache/zookeeper/zookeeper-jute/3.7.2/zookeeper-jute-3.7.2.jar \ + ad15d812b1f01f373638443adcda0e23fda549d65ced2be8c4f64bef33b5d774 + +fetch_jar netty-codec-http-4.1.135.Final.jar \ + io/netty/netty-codec-http/4.1.135.Final/netty-codec-http-4.1.135.Final.jar \ + 4018529d3d6aecf4044b98c75d9a90c91839ddf49c7aa484c5ac81c90a15da02