Verify SQLite dependencies

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -32,6 +32,37 @@ jobs:
             libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
             libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
             libffi-dev liblzma-dev
+      - name: Scan SQLite vulnerabilities with grype
+        run: |
+          # Install grype and required tools
+          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
+          sudo apt-get update && sudo apt-get install -y jq lsb-release
+
+          # Detect OS distribution info
+          DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
+          DISTRO_VERSION=$(lsb_release -sr)
+          echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION"
+
+          # Update grype vulnerability database
+          grype db update
+
+          # Check SQLite vulnerabilities in installed packages
+          echo "Scanning SQLite packages for vulnerabilities..."
+          GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
+          echo "Raw grype output:"
+          echo "$GRYPE_RAW_OUTPUT"
+
+          SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
+
+          if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
+            echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
+            echo "SQLite vulnerability details:"
+            echo "$SQLITE_SCAN_OUTPUT"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found"
+          fi
+        continue-on-error: false
       - name: Setup pyenv
         run: |
           curl https://pyenv.run | bash
@@ -135,6 +166,38 @@ jobs:
           bash ./chdb/build/build_static_lib.sh
           pyenv shell --unset
         continue-on-error: false
+      - name: Scan chdb libraries with grype
+        run: |
+          echo "Scanning chdb libraries for vulnerabilities..."
+
+          # Files to scan
+          FILES_TO_SCAN=""
+          [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
+          [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
+          FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)"
+
+          SQLITE_VULNERABILITIES_FOUND=false
+
+          for file in $FILES_TO_SCAN; do
+            if [ -f "$file" ]; then
+              echo "=== Scanning $file ==="
+              SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
+              echo "$SCAN_OUTPUT"
+
+              if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
+                echo "❌ SQLite vulnerability found in $file"
+                SQLITE_VULNERABILITIES_FOUND=true
+              fi
+            fi
+          done
+
+          if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
+            echo "❌ SQLite vulnerabilities detected in chdb libraries!"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found in chdb libraries"
+          fi
+        continue-on-error: false
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh

.github/workflows/build_linux_x86_wheels.yml

@@ -32,6 +32,37 @@ jobs:
             libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \
             libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \
             libffi-dev liblzma-dev
+      - name: Scan SQLite vulnerabilities with grype
+        run: |
+          # Install grype and required tools
+          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
+          sudo apt-get update && sudo apt-get install -y jq lsb-release
+
+          # Detect OS distribution info
+          DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
+          DISTRO_VERSION=$(lsb_release -sr)
+          echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION"
+
+          # Update grype vulnerability database
+          grype db update
+
+          # Check SQLite vulnerabilities in installed packages
+          echo "Scanning SQLite packages for vulnerabilities..."
+          GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true)
+          echo "Raw grype output:"
+          echo "$GRYPE_RAW_OUTPUT"
+
+          SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
+
+          if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
+            echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
+            echo "SQLite vulnerability details:"
+            echo "$SQLITE_SCAN_OUTPUT"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found"
+          fi
+        continue-on-error: false
       - name: Setup pyenv
         run: |
           curl https://pyenv.run | bash
@@ -135,6 +166,38 @@ jobs:
           bash ./chdb/build/build_static_lib.sh
           pyenv shell --unset
         continue-on-error: false
+      - name: Scan chdb libraries with grype
+        run: |
+          echo "Scanning chdb libraries for vulnerabilities..."
+
+          # Files to scan
+          FILES_TO_SCAN=""
+          [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
+          [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
+          FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)"
+
+          SQLITE_VULNERABILITIES_FOUND=false
+
+          for file in $FILES_TO_SCAN; do
+            if [ -f "$file" ]; then
+              echo "=== Scanning $file ==="
+              SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
+              echo "$SCAN_OUTPUT"
+
+              if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
+                echo "❌ SQLite vulnerability found in $file"
+                SQLITE_VULNERABILITIES_FOUND=true
+              fi
+            fi
+          done
+
+          if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
+            echo "❌ SQLite vulnerabilities detected in chdb libraries!"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found in chdb libraries"
+          fi
+        continue-on-error: false
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh

.github/workflows/build_macos_arm64_wheels.yml

@@ -76,30 +76,57 @@ jobs:
           uname -a
           export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1
           brew update
-          brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd
+          brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd
+          brew install openssl@3 || echo "OpenSSL install failed, continuing..."
           brew install --ignore-dependencies llvm@19
-          brew install git ninja libtool gettext gcc binutils grep findutils nasm
-          brew install --build-from-source ccache
+          brew install git ninja libtool gettext binutils grep findutils nasm
+          # brew install gcc || echo "GCC install failed, continuing..."
+          # brew install ccache || echo "ccache installation failed, continuing without it"
           brew install go
           cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
           export PATH=$(brew --prefix llvm@19)/bin:$PATH
           which clang++
           clang++ --version
           which go
           go version
-          ccache -s
+          ccache -s || echo "ccache not available yet"
+      - name: Scan SQLite vulnerabilities with grype
+        run: |
+          # Install grype
+          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
+
+          # Update grype vulnerability database
+          grype db update
+
+          # Check SQLite vulnerabilities in Homebrew packages
+          echo "Scanning SQLite packages for vulnerabilities..."
+          GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true)
+          echo "Raw grype output:"
+          echo "$GRYPE_RAW_OUTPUT"
+
+          SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
+
+          if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
+            echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
+            echo "SQLite vulnerability details:"
+            echo "$SQLITE_SCAN_OUTPUT"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found"
+          fi
+        continue-on-error: false
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Update submodules
         run: |
           git submodule update --init --recursive --jobs 4
-      - name: ccache
-        uses: hendrikmuhs/ccache-action@v1.2
-        with:
-          key: macos-13-xlarge
-          max-size: 5G
-          append-timestamp: true
+      # - name: ccache
+      #   uses: hendrikmuhs/ccache-action@v1.2
+      #   with:
+      #     key: macos-13-xlarge
+      #     max-size: 5G
+      #     append-timestamp: true
       - name: Run chdb/build.sh
         timeout-minutes: 600
         run: |
@@ -138,24 +165,56 @@ jobs:
           cat buildlib/pychdb_cmd.sh
           echo "libchdb_cmd.sh:"
           cat buildlib/libchdb_cmd.sh
-      - name: Run libchdb stub in examples dir
-        run: |
-          bash -x ./examples/runStub.sh
-      - name: Keep killall ccache and wait for ccache to finish
-        if: always()
+      - name: Scan chdb libraries with grype
         run: |
-          sleep 60
-          while ps -ef | grep ccache | grep -v grep; do \
-            killall ccache; \
-            sleep 10; \
+          echo "Scanning chdb libraries for vulnerabilities..."
+
+          # Files to scan
+          FILES_TO_SCAN=""
+          [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
+          [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
+          FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)"
+
+          SQLITE_VULNERABILITIES_FOUND=false
+
+          for file in $FILES_TO_SCAN; do
+            if [ -f "$file" ]; then
+              echo "=== Scanning $file ==="
+              SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
+              echo "$SCAN_OUTPUT"
+
+              if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
+                echo "❌ SQLite vulnerability found in $file"
+                SQLITE_VULNERABILITIES_FOUND=true
+              fi
+            fi
           done
-      - name: Check ccache statistics
+
+          if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
+            echo "❌ SQLite vulnerabilities detected in chdb libraries!"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found in chdb libraries"
+          fi
+        continue-on-error: false
+      - name: Run libchdb stub in examples dir
         run: |
-          ccache -s
-          ls -lh chdb
-          df -h
-        env:
-          CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++"
+          bash -x ./examples/runStub.sh
+      # - name: Keep killall ccache and wait for ccache to finish
+      #   if: always()
+      #   run: |
+      #     sleep 60
+      #     while ps -ef | grep ccache | grep -v grep; do \
+      #       killall ccache || true; \
+      #       sleep 10; \
+      #     done
+      # - name: Check ccache statistics
+      #   run: |
+      #     ccache -s || echo "ccache not available"
+      #     ls -lh chdb
+      #     df -h
+      #   env:
+      #     CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@19)/bin/clang CXX=$(brew --prefix llvm@19)/bin/clang++"
       - name: Build wheels
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"

.github/workflows/build_macos_x86_wheels.yml

@@ -22,7 +22,7 @@ on:
 jobs:
   build_universal_wheel:
     name: Build Universal Wheel (macOS x86_64)
-    runs-on: macos-13
+    runs-on: macos-14-large
     steps:
       - name: Setup pyenv
         run: |
@@ -79,7 +79,7 @@ jobs:
           brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd
           brew install --ignore-dependencies llvm@19
           brew install git ninja libtool gettext gcc binutils grep findutils nasm
-          brew install --build-from-source ccache
+          brew install ccache || echo "ccache installation failed, continuing without it"
           brew install go
           cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
           export PATH=$(brew --prefix llvm@19)/bin:$PATH
@@ -88,6 +88,31 @@ jobs:
           which go
           go version
           ccache -s
+      - name: Scan SQLite vulnerabilities with grype
+        run: |
+          # Install grype
+          curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
+
+          # Update grype vulnerability database
+          grype db update
+
+          # Check SQLite vulnerabilities in Homebrew packages
+          echo "Scanning SQLite packages for vulnerabilities..."
+          GRYPE_RAW_OUTPUT=$(grype dir:/usr/local --scope all-layers 2>/dev/null || true)
+          echo "Raw grype output:"
+          echo "$GRYPE_RAW_OUTPUT"
+
+          SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
+
+          if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
+            echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed."
+            echo "SQLite vulnerability details:"
+            echo "$SQLITE_SCAN_OUTPUT"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found"
+          fi
+        continue-on-error: false
       - uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -97,7 +122,7 @@ jobs:
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2
         with:
-          key: macos-13-x86_64
+          key: macos-14-x86_64
           max-size: 5G
           append-timestamp: true
       - name: Run chdb/build.sh
@@ -139,6 +164,38 @@ jobs:
           cat buildlib/pychdb_cmd.sh
           echo "libchdb_cmd.sh:"
           cat buildlib/libchdb_cmd.sh
+      - name: Scan chdb libraries with grype
+        run: |
+          echo "Scanning chdb libraries for vulnerabilities..."
+
+          # Files to scan
+          FILES_TO_SCAN=""
+          [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so"
+          [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a"
+          FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)"
+
+          SQLITE_VULNERABILITIES_FOUND=false
+
+          for file in $FILES_TO_SCAN; do
+            if [ -f "$file" ]; then
+              echo "=== Scanning $file ==="
+              SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
+              echo "$SCAN_OUTPUT"
+
+              if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
+                echo "❌ SQLite vulnerability found in $file"
+                SQLITE_VULNERABILITIES_FOUND=true
+              fi
+            fi
+          done
+
+          if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
+            echo "❌ SQLite vulnerabilities detected in chdb libraries!"
+            exit 1
+          else
+            echo "✅ No SQLite vulnerabilities found in chdb libraries"
+          fi
+        continue-on-error: false
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh