Merge pull request #182 from chdb-io/prAction

Run fast build and linter and tests on self hosted machine on Pull Request

.github/workflows/build_wheels.yml

@@ -8,10 +8,17 @@ on:
         required: true
   release:
     types: [created]
-  pull_request:
-    types: [opened, reopened]
-    paths-ignore:
-      - '**/.md'
+  # push:
+  #   branches:
+  #     - main
+  #   paths-ignore:
+  #     - '**/*.md'
+  # pull_request:
+  #   branches:
+  #     - main
+  #   paths-ignore:
+  #     - '**/*.md'
+
 
 jobs:
   build_wheels_linux:

.github/workflows/pr_ci.yaml

@@ -0,0 +1,75 @@
+name: Pull-CI
+
+on:
+  pull_request:
+    types: [opened, synchronize]
+
+jobs:
+  build:
+    env:
+      PYTHON_VERSIONS: "3.11"
+
+    runs-on: self-hosted
+    steps:
+      - name: Check for chdb directory
+        run: |
+          if [ ! -d "/home/ubuntu/pr_runner/chdb" ]; then
+            echo "chdb directory does not exist. Checkout the repository."
+            mkdir -p /home/ubuntu/pr_runner/
+            git clone https://github.com/chdb-io/chdb.git /home/ubuntu/pr_runner/chdb
+          fi
+      
+      - name: Check for ccache status
+        run: |
+          ccache -sv
+
+      - name: Copy submodules
+        run: cp -a /builder_cache/contrib /home/ubuntu/pr_runner/chdb/
+
+      - name: Cleanup and update chdb directory
+        run: |
+          cd /home/ubuntu/pr_runner/chdb
+          git fetch origin || true
+          git reset --hard origin/${{ github.head_ref }} || true
+          git clean -fdx || true
+          git checkout -f --progress ${{ github.head_ref }} || true
+          git status -v || true
+        continue-on-error: true
+
+      - name: Code style check
+        run: |
+          export PYENV_ROOT="$HOME/.pyenv"
+          [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
+          eval "$(pyenv init -)"
+          pyenv local 3.11
+          python3 -m pip install flake8
+          cd chdb && python3 -m flake8
+        working-directory: /home/ubuntu/pr_runner/chdb
+
+      - name: Cleanup dist directory
+        run: rm -rf /home/ubuntu/pr_runner/chdb/dist/*
+
+      - name: Set PYTHON_VERSIONS environment variable
+        run: echo "PYTHON_VERSIONS=3.11" >> $GITHUB_ENV
+
+      - name: Run build script
+        run: bash -x ./chdb/build_linux_arm64.sh
+        working-directory: /home/ubuntu/pr_runner/chdb
+
+      - name: Check ccache statistics
+        run: |
+          ccache -s
+          ls -lh chdb
+          df -h
+        working-directory: /home/ubuntu/pr_runner/chdb
+
+      - name: Audit wheels
+        run: |
+          export PYENV_ROOT="$HOME/.pyenv"
+          [[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
+          eval "$(pyenv init -)"
+          pyenv local 3.11
+          ls -lh dist
+          python3 -m pip install auditwheel
+          python3 -m auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl
+        working-directory: /home/ubuntu/pr_runner/chdb

chdb/.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 130
+extend-ignore = E722

chdb/__init__.py

@@ -2,11 +2,15 @@
 import os
 
 
+class ChdbError(Exception):
+    """Base class for exceptions in this module."""
+
+
 _arrow_format = set({"dataframe", "arrowtable"})
 _process_result_format_funs = {
-        "dataframe" : lambda x : to_df(x),
-        "arrowtable": lambda x : to_arrowTable(x)
-        }
+    "dataframe": lambda x: to_df(x),
+    "arrowtable": lambda x: to_arrowTable(x)
+}
 
 # If any UDF is defined, the path of the UDF will be set to this variable
 # and the path will be deleted when the process exits
@@ -33,7 +37,7 @@
     # Change here if project is renamed and does not equal the package name
     dist_name = __name__
     __version__ = ".".join(map(str, chdb_version))
-except:  # pragma: no cover
+except:  # noqa
     __version__ = "unknown"
 
 
@@ -42,8 +46,8 @@ def to_arrowTable(res):
     """convert res to arrow table"""
     # try import pyarrow and pandas, if failed, raise ImportError with suggestion
     try:
-        import pyarrow as pa
-        import pandas
+        import pyarrow as pa  # noqa
+        import pandas as pd  # noqa
     except ImportError as e:
         print(f"ImportError: {e}")
         print('Please install pyarrow and pandas via "pip install pyarrow pandas"')
@@ -66,10 +70,14 @@ def query(sql, output_format="CSV", path="", udf_path=""):
     if udf_path != "":
         g_udf_path = udf_path
     lower_output_format = output_format.lower()
-    result_func = _process_result_format_funs.get(lower_output_format, lambda x : x)
+    result_func = _process_result_format_funs.get(lower_output_format, lambda x: x)
     if lower_output_format in _arrow_format:
         output_format = "Arrow"
     res = _chdb.query(sql, output_format, path=path, udf_path=g_udf_path)
     if res.has_error():
-        raise Exception(res.error_message())
+        raise ChdbError(res.error_message())
     return result_func(res)
+
+
+__all__ = ["ChdbError", "query", "chdb_version",
+           "engine_version", "to_df", "to_arrowTable"]