From 02bc4359d95c02799cec19a9baecf5546c621de0 Mon Sep 17 00:00:00 2001
From: harshsoni2024 <harshsoni2024@gmail.com>
Date: Fri, 31 Jul 2026 14:41:08 +0530
Subject: [PATCH 1/5] fix(ingestion): patch CVE jars in ingestion-slim image
 and dedupe PySpark jar surgery
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The ingestion-slim image (ingestion/operators/docker/Dockerfile*) shipped
vulnerable jars bundled by PySpark 3.5.6 that the earlier CVE fix (#30516)
never patched — that fix only touched the airflow image (ingestion/Dockerfile*),
so the PySpark jar surgery never reached the slim image. AWS Inspector kept
flagging zookeeper, jackson-asl, netty, and a phantom black finding on it.

Extract the jar patch into a single SHA256-pinned script and run it in all four
ingestion Dockerfiles so the two images can no longer drift:

  - zookeeper 3.6.3 -> 3.7.2          CVE-2023-44981 (SASL quorum auth bypass)
  - jackson-mapper/core-asl 1.9.13    CVE-2019-10202 (removed; no upstream fix)
  - netty-codec-http 4.1.96 -> 4.1.135  CVE-2026-42581 / CVE-2026-42584
    (both fixed in 4.1.133.Final; staying on the 4.1.x line keeps binary
    compatibility with PySpark's sibling netty 4.1.96 jars — verified pyspark
    and SparkSession still import)

Also strip spacy's bundled tests/package/requirements.txt from the slim images:
it pins an old black that scanners misreport as an installed package
(CVE-2026-31900). The file is test-only and never imported at runtime.

Left as documented residuals (no drop-in fix for this runtime):
  - derby 10.14.2.0 (CVE-2022-46337): only fix needs Java 21, image is Java 17;
    vulnerable LDAP path unused
  - jetty 9.4.x (CVE-2026-2332): shaded inside hadoop-client-runtime / spark-core
    uber-jars, not a swappable standalone jar; clears on a future pyspark bump

Verified against a local build of the slim CI image: patched jars present,
jackson-asl and the black fixture gone, pyspark imports cleanly.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 ingestion/Dockerfile                     | 34 ++----------
 ingestion/Dockerfile.ci                  | 34 ++----------
 ingestion/operators/docker/Dockerfile    | 10 ++++
 ingestion/operators/docker/Dockerfile.ci | 10 ++++
 ingestion/scripts/patch_pyspark_jars.sh  | 66 ++++++++++++++++++++++++
 5 files changed, 94 insertions(+), 60 deletions(-)
 create mode 100755 ingestion/scripts/patch_pyspark_jars.sh

diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile
index 5018b4f4a318..362d66a016d0 100644
--- a/ingestion/Dockerfile
+++ b/ingestion/Dockerfile
@@ -107,36 +107,10 @@ RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9"
 RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.2.2/constraints-3.10.txt"
 RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}"
 
-# Patch vulnerable Hive-metastore jars bundled inside PySpark 3.5.6 (the Deltalake
-# connector calls .enableHiveSupport()). SHA256-pinned to Maven Central.
-#   zookeeper 3.6.3 -> 3.7.2        CVE-2023-44981 (SASL quorum auth bypass); ZK 3.7 runs on Java 8+
-#   jackson-mapper/core-asl 1.9.13  CVE-2019-10202 (deserialization RCE) -- removed, no fix upstream
-# Derby (CVE-2022-46337) is deliberately left at the bundled 10.14.2.0. The only fixed
-# release on Maven Central is 10.17.1.0, which requires Java 21 — this image ships Java 17
-# (default-jre-headless), and the Java-8/11/17 backport jars (10.14.3.0 / 10.15.2.1 /
-# 10.16.1.2) were never published upstream, so no drop-in Java-17 fix exists. The CVE is
-# an LDAP-authenticator injection; OM's embedded Derby metastore uses no LDAP auth, so the
-# vulnerable path is unreachable. Revisit if PySpark bumps Derby or the image moves to Java 21.
-# Skips cleanly when pyspark is absent (e.g. an INGESTION_DEPENDENCY build without
-# the deltalake/pyspark deps); only patches jars when the pyspark jars dir exists.
-RUN JARS_DIR="$(python -c 'import os,pyspark;print(os.path.join(os.path.dirname(pyspark.__file__),"jars"))' 2>/dev/null || true)" \
-  && if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then \
-       echo "pyspark not installed; skipping Hive-metastore jar patch"; \
-     else \
-       fetch_jar() { \
-         wget -q "https://repo1.maven.org/maven2/$2" -O "$1" \
-         && echo "$3  $1" | sha256sum -c - || exit 1; \
-       }; \
-       cd "${JARS_DIR}" \
-       && rm -f zookeeper-*.jar zookeeper-jute-*.jar \
-                jackson-mapper-asl-*.jar jackson-core-asl-*.jar \
-       && fetch_jar zookeeper-3.7.2.jar \
-            org/apache/zookeeper/zookeeper/3.7.2/zookeeper-3.7.2.jar \
-            b12d6fb4afd7b3849d3a9a5a38b9260c23a12e1ea58ca8c8d775880249cb8eac \
-       && fetch_jar zookeeper-jute-3.7.2.jar \
-            org/apache/zookeeper/zookeeper-jute/3.7.2/zookeeper-jute-3.7.2.jar \
-            ad15d812b1f01f373638443adcda0e23fda549d65ced2be8c4f64bef33b5d774; \
-     fi
+# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+COPY --chown=airflow:0 ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
+RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
 
 # Temporary workaround for https://github.com/open-metadata/OpenMetadata/issues/9593
 RUN [ $(uname -m) = "x86_64" ] \
diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci
index 42f79c106d86..096cd3df7314 100644
--- a/ingestion/Dockerfile.ci
+++ b/ingestion/Dockerfile.ci
@@ -144,36 +144,10 @@ RUN [ $(uname -m) = "x86_64" ] \
   && pip install ".[db2]" \
   || echo "DB2 not supported on ARM architectures."
 
-# Patch vulnerable Hive-metastore jars bundled inside PySpark 3.5.6 (the Deltalake
-# connector calls .enableHiveSupport()). SHA256-pinned to Maven Central.
-#   zookeeper 3.6.3 -> 3.7.2        CVE-2023-44981 (SASL quorum auth bypass); ZK 3.7 runs on Java 8+
-#   jackson-mapper/core-asl 1.9.13  CVE-2019-10202 (deserialization RCE) -- removed, no fix upstream
-# Derby (CVE-2022-46337) is deliberately left at the bundled 10.14.2.0. The only fixed
-# release on Maven Central is 10.17.1.0, which requires Java 21 — this image ships Java 17
-# (default-jre-headless), and the Java-8/11/17 backport jars (10.14.3.0 / 10.15.2.1 /
-# 10.16.1.2) were never published upstream, so no drop-in Java-17 fix exists. The CVE is
-# an LDAP-authenticator injection; OM's embedded Derby metastore uses no LDAP auth, so the
-# vulnerable path is unreachable. Revisit if PySpark bumps Derby or the image moves to Java 21.
-# Skips cleanly when pyspark is absent (e.g. an INGESTION_DEPENDENCY build without
-# the deltalake/pyspark deps); only patches jars when the pyspark jars dir exists.
-RUN JARS_DIR="$(python -c 'import os,pyspark;print(os.path.join(os.path.dirname(pyspark.__file__),"jars"))' 2>/dev/null || true)" \
-  && if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then \
-       echo "pyspark not installed; skipping Hive-metastore jar patch"; \
-     else \
-       fetch_jar() { \
-         wget -q "https://repo1.maven.org/maven2/$2" -O "$1" \
-         && echo "$3  $1" | sha256sum -c - || exit 1; \
-       }; \
-       cd "${JARS_DIR}" \
-       && rm -f zookeeper-*.jar zookeeper-jute-*.jar \
-                jackson-mapper-asl-*.jar jackson-core-asl-*.jar \
-       && fetch_jar zookeeper-3.7.2.jar \
-            org/apache/zookeeper/zookeeper/3.7.2/zookeeper-3.7.2.jar \
-            b12d6fb4afd7b3849d3a9a5a38b9260c23a12e1ea58ca8c8d775880249cb8eac \
-       && fetch_jar zookeeper-jute-3.7.2.jar \
-            org/apache/zookeeper/zookeeper-jute/3.7.2/zookeeper-jute-3.7.2.jar \
-            ad15d812b1f01f373638443adcda0e23fda549d65ced2be8c4f64bef33b5d774; \
-     fi
+# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+COPY --chown=airflow:0 ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
+RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
 
 # bump python-daemon for https://github.com/apache/airflow/pull/29916
 RUN pip install "python-daemon>=3.0.0"
diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile
index c79a5b8a0963..c0fac5d94771 100644
--- a/ingestion/operators/docker/Dockerfile
+++ b/ingestion/operators/docker/Dockerfile
@@ -137,6 +137,16 @@ RUN [ $(uname -m) = "x86_64" ] \
   && pip install "openmetadata-ingestion[db2]~=${RI_VERSION}" \
   || echo "DB2 not supported on ARM architectures."
 
+# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+COPY --chown=openmetadata:openmetadata ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
+RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
+
+# Strip spacy's bundled CI test fixture: spacy/tests/package/requirements.txt pins an old
+# black, which image scanners misreport as an installed package (CVE-2026-31900). The file
+# is test-only, never imported at runtime. Guarded so builds without spacy don't fail.
+RUN find /home/openmetadata/.local/lib/python*/site-packages/spacy/tests -name 'requirements.txt' -delete 2>/dev/null || true
+
 # Uninstalling psycopg2-binary and installing psycopg2 instead
 # because the psycopg2-binary generates a architecture specific error
 # while authenticating connection with the airflow, psycopg2 solves this error
diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci
index 01e0f02695a1..6cdc8fa3a4b3 100644
--- a/ingestion/operators/docker/Dockerfile.ci
+++ b/ingestion/operators/docker/Dockerfile.ci
@@ -142,6 +142,16 @@ RUN [ $(uname -m) = "x86_64" ] \
   && pip install ".[db2]" \
   || echo "DB2 not supported on ARM architectures."
 
+# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+COPY --chown=openmetadata:openmetadata ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
+RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
+
+# Strip spacy's bundled CI test fixture: spacy/tests/package/requirements.txt pins an old
+# black, which image scanners misreport as an installed package (CVE-2026-31900). The file
+# is test-only, never imported at runtime. Guarded so builds without spacy don't fail.
+RUN find /home/openmetadata/.local/lib/python*/site-packages/spacy/tests -name 'requirements.txt' -delete 2>/dev/null || true
+
 # Required for Airflow DockerOperator, as we need to run the workflows from a `python main.py` command in the container.
 COPY ingestion/operators/docker/*.py .
 
diff --git a/ingestion/scripts/patch_pyspark_jars.sh b/ingestion/scripts/patch_pyspark_jars.sh
new file mode 100755
index 000000000000..bc11f9af5302
--- /dev/null
+++ b/ingestion/scripts/patch_pyspark_jars.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# Patch vulnerable jars bundled inside PySpark 3.5.6 (the Deltalake connector calls
+# .enableHiveSupport()). All replacement jars are SHA256-pinned to Maven Central.
+#
+#   zookeeper 3.6.3 -> 3.7.2         CVE-2023-44981 (SASL quorum auth bypass); ZK 3.7 runs on Java 8+
+#   jackson-mapper/core-asl 1.9.13   CVE-2019-10202 (deserialization RCE) -- removed, no fix upstream
+#   netty-codec-http 4.1.96 -> 4.1.135  CVE-2026-42581 / CVE-2026-42584 (HTTP request smuggling);
+#                                    both fixed in 4.1.133.Final, so staying on the 4.1.x line keeps
+#                                    binary compatibility with PySpark's sibling netty 4.1.96 jars.
+#
+# Derby (CVE-2022-46337) is deliberately left at the bundled 10.14.2.0. The only fixed
+# release on Maven Central is 10.17.1.0, which requires Java 21 — these images ship Java 17
+# (default-jre-headless), and the Java-8/11/17 backport jars (10.14.3.0 / 10.15.2.1 /
+# 10.16.1.2) were never published upstream, so no drop-in Java-17 fix exists. The CVE is an
+# LDAP-authenticator injection; OM's embedded Derby metastore uses no LDAP auth, so the
+# vulnerable path is unreachable. Revisit if PySpark bumps Derby or the images move to Java 21.
+#
+# jetty-http (CVE-2026-2332) is shaded inside PySpark's hadoop-client-runtime / spark-core
+# uber-jars, so it cannot be swapped as a standalone jar without repackaging Spark. Left as a
+# documented residual.
+#
+# Skips cleanly when pyspark is absent (e.g. an INGESTION_DEPENDENCY build without the
+# deltalake/pyspark deps); only patches jars when the pyspark jars dir exists.
+
+set -euo pipefail
+
+JARS_DIR="$(python -c 'import os,pyspark;print(os.path.join(os.path.dirname(pyspark.__file__),"jars"))' 2>/dev/null || true)"
+
+if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then
+  echo "pyspark not installed; skipping PySpark jar patch"
+  exit 0
+fi
+
+fetch_jar() {
+  wget -q "https://repo1.maven.org/maven2/$2" -O "$1"
+  echo "$3  $1" | sha256sum -c -
+}
+
+cd "${JARS_DIR}"
+
+rm -f zookeeper-*.jar zookeeper-jute-*.jar \
+      jackson-mapper-asl-*.jar jackson-core-asl-*.jar \
+      netty-codec-http-*.jar
+
+fetch_jar zookeeper-3.7.2.jar \
+  org/apache/zookeeper/zookeeper/3.7.2/zookeeper-3.7.2.jar \
+  b12d6fb4afd7b3849d3a9a5a38b9260c23a12e1ea58ca8c8d775880249cb8eac
+
+fetch_jar zookeeper-jute-3.7.2.jar \
+  org/apache/zookeeper/zookeeper-jute/3.7.2/zookeeper-jute-3.7.2.jar \
+  ad15d812b1f01f373638443adcda0e23fda549d65ced2be8c4f64bef33b5d774
+
+fetch_jar netty-codec-http-4.1.135.Final.jar \
+  io/netty/netty-codec-http/4.1.135.Final/netty-codec-http-4.1.135.Final.jar \
+  4018529d3d6aecf4044b98c75d9a90c91839ddf49c7aa484c5ac81c90a15da02

From 4ae59273e5ec29fe908c32ef7980e67938de538a Mon Sep 17 00:00:00 2001
From: harshsoni2024 <harshsoni2024@gmail.com>
Date: Fri, 31 Jul 2026 18:05:08 +0530
Subject: [PATCH 2/5] fix(ingestion): fail-closed pyspark detection and fold
 spaCy fixture cleanup into jar patch

Address review feedback on the PySpark CVE-jar remediation:

- Detection no longer fails open. Previously `python -c 'import pyspark' 2>/dev/null
  || true` treated a broken-but-installed pyspark the same as an absent one, letting
  an image with unpatched jars ship silently. The script now distinguishes three
  cases via exit code: genuinely absent (skip), installed-but-unimportable (fail the
  build), importable (patch).

- Fold the spaCy test-fixture cleanup (spacy/tests/**/requirements.txt, the phantom
  black CVE-2026-31900) into patch_pyspark_jars.sh so it runs wherever the patch runs,
  and drop the separate per-Dockerfile `find ... -delete` RUN lines that could drift.

Self-guarding and idempotent: no-op when pyspark is absent (INGESTION_DEPENDENCY=slim
builds), active when present (default `all`).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 ingestion/Dockerfile                     |  6 ++-
 ingestion/Dockerfile.ci                  |  6 ++-
 ingestion/operators/docker/Dockerfile    | 11 ++----
 ingestion/operators/docker/Dockerfile.ci | 11 ++----
 ingestion/scripts/patch_pyspark_jars.sh  | 50 ++++++++++++++++++++++--
 5 files changed, 62 insertions(+), 22 deletions(-)

diff --git a/ingestion/Dockerfile b/ingestion/Dockerfile
index 362d66a016d0..8c91c51cf2d9 100644
--- a/ingestion/Dockerfile
+++ b/ingestion/Dockerfile
@@ -107,8 +107,10 @@ RUN pip install --no-build-isolation "cx_Oracle>=8.3.0,<9"
 RUN pip install "openmetadata-managed-apis~=${RI_VERSION}" --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.2.2/constraints-3.10.txt"
 RUN pip install "openmetadata-ingestion[${INGESTION_DEPENDENCY}]~=${RI_VERSION}"
 
-# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
-# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture.
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after the
+# final pip install; fails the build on a broken pyspark install rather than shipping
+# unpatched jars.
 COPY --chown=airflow:0 ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
 RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
 
diff --git a/ingestion/Dockerfile.ci b/ingestion/Dockerfile.ci
index 096cd3df7314..6092cbd1e835 100644
--- a/ingestion/Dockerfile.ci
+++ b/ingestion/Dockerfile.ci
@@ -144,8 +144,10 @@ RUN [ $(uname -m) = "x86_64" ] \
   && pip install ".[db2]" \
   || echo "DB2 not supported on ARM architectures."
 
-# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
-# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture.
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after the
+# final pip install; fails the build on a broken pyspark install rather than shipping
+# unpatched jars.
 COPY --chown=airflow:0 ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
 RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
 
diff --git a/ingestion/operators/docker/Dockerfile b/ingestion/operators/docker/Dockerfile
index c0fac5d94771..e447dad3bd53 100644
--- a/ingestion/operators/docker/Dockerfile
+++ b/ingestion/operators/docker/Dockerfile
@@ -137,16 +137,13 @@ RUN [ $(uname -m) = "x86_64" ] \
   && pip install "openmetadata-ingestion[db2]~=${RI_VERSION}" \
   || echo "DB2 not supported on ARM architectures."
 
-# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
-# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture.
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after all
+# pip installs so the pyspark/spacy trees exist; the script fails the build on a broken
+# pyspark install rather than silently shipping unpatched jars.
 COPY --chown=openmetadata:openmetadata ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
 RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
 
-# Strip spacy's bundled CI test fixture: spacy/tests/package/requirements.txt pins an old
-# black, which image scanners misreport as an installed package (CVE-2026-31900). The file
-# is test-only, never imported at runtime. Guarded so builds without spacy don't fail.
-RUN find /home/openmetadata/.local/lib/python*/site-packages/spacy/tests -name 'requirements.txt' -delete 2>/dev/null || true
-
 # Uninstalling psycopg2-binary and installing psycopg2 instead
 # because the psycopg2-binary generates a architecture specific error
 # while authenticating connection with the airflow, psycopg2 solves this error
diff --git a/ingestion/operators/docker/Dockerfile.ci b/ingestion/operators/docker/Dockerfile.ci
index 6cdc8fa3a4b3..76c81bf903e4 100644
--- a/ingestion/operators/docker/Dockerfile.ci
+++ b/ingestion/operators/docker/Dockerfile.ci
@@ -142,16 +142,13 @@ RUN [ $(uname -m) = "x86_64" ] \
   && pip install ".[db2]" \
   || echo "DB2 not supported on ARM architectures."
 
-# Patch CVE-vulnerable jars bundled inside PySpark (zookeeper, jackson-asl, netty).
-# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale.
+# Patch CVE-vulnerable jars bundled inside PySpark and strip the spaCy scanner fixture.
+# See ingestion/scripts/patch_pyspark_jars.sh for the per-CVE rationale. Runs after all
+# pip installs so the pyspark/spacy trees exist; the script fails the build on a broken
+# pyspark install rather than silently shipping unpatched jars.
 COPY --chown=openmetadata:openmetadata ingestion/scripts/patch_pyspark_jars.sh /tmp/patch_pyspark_jars.sh
 RUN bash /tmp/patch_pyspark_jars.sh && rm -f /tmp/patch_pyspark_jars.sh
 
-# Strip spacy's bundled CI test fixture: spacy/tests/package/requirements.txt pins an old
-# black, which image scanners misreport as an installed package (CVE-2026-31900). The file
-# is test-only, never imported at runtime. Guarded so builds without spacy don't fail.
-RUN find /home/openmetadata/.local/lib/python*/site-packages/spacy/tests -name 'requirements.txt' -delete 2>/dev/null || true
-
 # Required for Airflow DockerOperator, as we need to run the workflows from a `python main.py` command in the container.
 COPY ingestion/operators/docker/*.py .
 
diff --git a/ingestion/scripts/patch_pyspark_jars.sh b/ingestion/scripts/patch_pyspark_jars.sh
index bc11f9af5302..ddaef9af5897 100755
--- a/ingestion/scripts/patch_pyspark_jars.sh
+++ b/ingestion/scripts/patch_pyspark_jars.sh
@@ -30,16 +30,48 @@
 # uber-jars, so it cannot be swapped as a standalone jar without repackaging Spark. Left as a
 # documented residual.
 #
-# Skips cleanly when pyspark is absent (e.g. an INGESTION_DEPENDENCY build without the
-# deltalake/pyspark deps); only patches jars when the pyspark jars dir exists.
+# Skips cleanly when pyspark is genuinely absent (e.g. an INGESTION_DEPENDENCY build
+# without the deltalake/pyspark deps). A pyspark that is installed but fails to import is
+# treated as an error and fails the build, so a broken install can never ship unpatched
+# jars silently.
 
 set -euo pipefail
 
-JARS_DIR="$(python -c 'import os,pyspark;print(os.path.join(os.path.dirname(pyspark.__file__),"jars"))' 2>/dev/null || true)"
+# Locate PySpark's bundled jars directory, distinguishing three cases via exit code:
+#   0 -> installed and importable (prints the jars dir on stdout)
+#   2 -> genuinely not installed (top-level pyspark module missing) -> skip
+#   3 -> installed but import failed for any other reason -> fail the build
+locate_pyspark_jars() {
+  python - <<'PY'
+import os, sys
+try:
+    import pyspark
+except ModuleNotFoundError as exc:
+    if exc.name == "pyspark" or (exc.name or "").split(".")[0] == "pyspark":
+        sys.exit(2)
+    # pyspark itself is present but one of ITS imports is missing -> broken install
+    sys.stderr.write(f"pyspark is installed but failed to import: {exc!r}\n")
+    sys.exit(3)
+except Exception as exc:
+    sys.stderr.write(f"pyspark is installed but failed to import: {exc!r}\n")
+    sys.exit(3)
+print(os.path.join(os.path.dirname(pyspark.__file__), "jars"))
+PY
+}
 
-if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then
+JARS_DIR="$(locate_pyspark_jars)" && rc=0 || rc=$?
+
+if [ "${rc}" -eq 2 ]; then
   echo "pyspark not installed; skipping PySpark jar patch"
   exit 0
+elif [ "${rc}" -ne 0 ]; then
+  echo "ERROR: pyspark is installed but not importable; refusing to ship unpatched jars" >&2
+  exit 1
+fi
+
+if [ -z "${JARS_DIR}" ] || [ ! -d "${JARS_DIR}" ]; then
+  echo "ERROR: pyspark imported but its jars dir '${JARS_DIR}' is missing" >&2
+  exit 1
 fi
 
 fetch_jar() {
@@ -64,3 +96,13 @@ fetch_jar zookeeper-jute-3.7.2.jar \
 fetch_jar netty-codec-http-4.1.135.Final.jar \
   io/netty/netty-codec-http/4.1.135.Final/netty-codec-http-4.1.135.Final.jar \
   4018529d3d6aecf4044b98c75d9a90c91839ddf49c7aa484c5ac81c90a15da02
+
+# Strip spacy's bundled CI test fixture. spacy/tests/package/requirements.txt pins an old
+# black, which image scanners misreport as an installed package (CVE-2026-31900). The file
+# is test-only and never imported at runtime. spacy is installed alongside pyspark in the
+# same (deltalake/all) extra, so this runs in the same place the jar patch does. Guarded so
+# a build without spacy does not fail.
+SPACY_DIR="$(python -c 'import os,spacy;print(os.path.dirname(spacy.__file__))' 2>/dev/null || true)"
+if [ -n "${SPACY_DIR}" ] && [ -d "${SPACY_DIR}/tests" ]; then
+  find "${SPACY_DIR}/tests" -name 'requirements.txt' -delete
+fi

From 30d3a6f2583bcbc7b37c36b392c750feb9c9e4d8 Mon Sep 17 00:00:00 2001
From: harshsoni2024 <harshsoni2024@gmail.com>
Date: Fri, 31 Jul 2026 18:33:00 +0530
Subject: [PATCH 3/5] fix(ingestion): run spaCy fixture cleanup on all paths
 and restore tolerant delete
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Address review feedback on patch_pyspark_jars.sh:

- The spaCy fixture cleanup was placed after the early `exit 0` that fires when
  PySpark is absent, so a build with spaCy but no PySpark (e.g.
  INGESTION_DEPENDENCY=pii-processor — spaCy comes from pii-processor/sample-data,
  PySpark from deltalake, independent extras) skipped the strip and reintroduced the
  CVE-2026-31900 phantom finding. Extract it into strip_spacy_scanner_fixture() and
  call it before the PySpark exit branches so it runs on every path.

- Restore the best-effort delete: `find ... -delete` under `set -euo pipefail` aborted
  the build on any undeletable file (read-only layer). Back to `-delete 2>/dev/null
  || true` so the fixture strip stays non-fatal.
]
---
 ingestion/scripts/patch_pyspark_jars.sh | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/ingestion/scripts/patch_pyspark_jars.sh b/ingestion/scripts/patch_pyspark_jars.sh
index ddaef9af5897..3bb37e38f7c9 100755
--- a/ingestion/scripts/patch_pyspark_jars.sh
+++ b/ingestion/scripts/patch_pyspark_jars.sh
@@ -59,6 +59,21 @@ print(os.path.join(os.path.dirname(pyspark.__file__), "jars"))
 PY
 }
 
+# Strip spacy's bundled CI test fixture. spacy/tests/package/requirements.txt pins an old
+# black, which image scanners misreport as an installed package (CVE-2026-31900). The file
+# is test-only and never imported at runtime. spaCy (from the pii-processor / sample-data
+# extras) installs independently of pyspark (from deltalake), so this must run on every
+# path — including builds that ship spaCy but not pyspark. Best-effort: never fails the build.
+strip_spacy_scanner_fixture() {
+  local spacy_dir
+  spacy_dir="$(python -c 'import os,spacy;print(os.path.dirname(spacy.__file__))' 2>/dev/null || true)"
+  if [ -n "${spacy_dir}" ] && [ -d "${spacy_dir}/tests" ]; then
+    find "${spacy_dir}/tests" -name 'requirements.txt' -delete 2>/dev/null || true
+  fi
+}
+
+strip_spacy_scanner_fixture
+
 JARS_DIR="$(locate_pyspark_jars)" && rc=0 || rc=$?
 
 if [ "${rc}" -eq 2 ]; then
@@ -96,13 +111,3 @@ fetch_jar zookeeper-jute-3.7.2.jar \
 fetch_jar netty-codec-http-4.1.135.Final.jar \
   io/netty/netty-codec-http/4.1.135.Final/netty-codec-http-4.1.135.Final.jar \
   4018529d3d6aecf4044b98c75d9a90c91839ddf49c7aa484c5ac81c90a15da02
-
-# Strip spacy's bundled CI test fixture. spacy/tests/package/requirements.txt pins an old
-# black, which image scanners misreport as an installed package (CVE-2026-31900). The file
-# is test-only and never imported at runtime. spacy is installed alongside pyspark in the
-# same (deltalake/all) extra, so this runs in the same place the jar patch does. Guarded so
-# a build without spacy does not fail.
-SPACY_DIR="$(python -c 'import os,spacy;print(os.path.dirname(spacy.__file__))' 2>/dev/null || true)"
-if [ -n "${SPACY_DIR}" ] && [ -d "${SPACY_DIR}/tests" ]; then
-  find "${SPACY_DIR}/tests" -name 'requirements.txt' -delete
-fi

From 2758bd0c493e11b8814df4d1aba8ad92b5473f0e Mon Sep 17 00:00:00 2001
From: harshsoni2024 <harshsoni2024@gmail.com>
Date: Fri, 31 Jul 2026 18:41:33 +0530
Subject: [PATCH 4/5] fix(ingestion): use find_spec so a broken pyspark
 submodule import fails closed instead of skipping

---
 ingestion/scripts/patch_pyspark_jars.sh | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/ingestion/scripts/patch_pyspark_jars.sh b/ingestion/scripts/patch_pyspark_jars.sh
index 3bb37e38f7c9..d6920f76a594 100755
--- a/ingestion/scripts/patch_pyspark_jars.sh
+++ b/ingestion/scripts/patch_pyspark_jars.sh
@@ -43,18 +43,26 @@ set -euo pipefail
 #   3 -> installed but import failed for any other reason -> fail the build
 locate_pyspark_jars() {
   python - <<'PY'
-import os, sys
+import importlib.util, os, sys
+
+# Distinguish "pyspark genuinely not installed" from "pyspark present but broken".
+# find_spec only inspects the top-level pyspark package without importing it, so a
+# missing pyspark.* submodule during a real import is never mistaken for absence.
 try:
-    import pyspark
-except ModuleNotFoundError as exc:
-    if exc.name == "pyspark" or (exc.name or "").split(".")[0] == "pyspark":
-        sys.exit(2)
-    # pyspark itself is present but one of ITS imports is missing -> broken install
+    spec = importlib.util.find_spec("pyspark")
+except Exception as exc:
     sys.stderr.write(f"pyspark is installed but failed to import: {exc!r}\n")
     sys.exit(3)
+
+if spec is None:
+    sys.exit(2)
+
+try:
+    import pyspark
 except Exception as exc:
     sys.stderr.write(f"pyspark is installed but failed to import: {exc!r}\n")
     sys.exit(3)
+
 print(os.path.join(os.path.dirname(pyspark.__file__), "jars"))
 PY
 }

From 0668f637ee85305713227d69241b5f5892148387 Mon Sep 17 00:00:00 2001
From: harshsoni2024 <harshsoni2024@gmail.com>
Date: Fri, 31 Jul 2026 19:33:34 +0530
Subject: [PATCH 5/5] fix: spacy cleanup

---
 ingestion/scripts/patch_pyspark_jars.sh | 28 +++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/ingestion/scripts/patch_pyspark_jars.sh b/ingestion/scripts/patch_pyspark_jars.sh
index d6920f76a594..4f7e8dec5c98 100755
--- a/ingestion/scripts/patch_pyspark_jars.sh
+++ b/ingestion/scripts/patch_pyspark_jars.sh
@@ -71,12 +71,32 @@ PY
 # black, which image scanners misreport as an installed package (CVE-2026-31900). The file
 # is test-only and never imported at runtime. spaCy (from the pii-processor / sample-data
 # extras) installs independently of pyspark (from deltalake), so this must run on every
-# path — including builds that ship spaCy but not pyspark. Best-effort: never fails the build.
+# path — including builds that ship spaCy but not pyspark.
+#
+# Deterministic, not best-effort: locate spacy via find_spec (without importing it, so an
+# installed-but-broken spacy is still found), delete the fixture, then assert it is gone.
+# The image scanner reads the file on disk regardless of whether spacy imports, so a broken
+# spacy must not let the fixture survive while the build succeeds.
 strip_spacy_scanner_fixture() {
   local spacy_dir
-  spacy_dir="$(python -c 'import os,spacy;print(os.path.dirname(spacy.__file__))' 2>/dev/null || true)"
-  if [ -n "${spacy_dir}" ] && [ -d "${spacy_dir}/tests" ]; then
-    find "${spacy_dir}/tests" -name 'requirements.txt' -delete 2>/dev/null || true
+  spacy_dir="$(python - <<'PY'
+import importlib.util, os, sys
+spec = importlib.util.find_spec("spacy")
+if spec is None or not spec.submodule_search_locations:
+    sys.exit(0)
+print(spec.submodule_search_locations[0])
+PY
+)"
+  if [ -z "${spacy_dir}" ] || [ ! -d "${spacy_dir}/tests" ]; then
+    return 0
+  fi
+  find "${spacy_dir}/tests" -name 'requirements.txt' -delete
+  local leftover
+  leftover="$(find "${spacy_dir}/tests" -name 'requirements.txt')"
+  if [ -n "${leftover}" ]; then
+    echo "ERROR: spaCy scanner fixture still present after delete:" >&2
+    echo "${leftover}" >&2
+    exit 1
   fi
 }