[python] Migrate PyPaimon packaging to pyproject.toml

.github/workflows/paimon-python-checks.yml

@@ -41,14 +41,14 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # Lint + test on 3.6 and 3.10 only (not every Python version).
+  # Lint + test on selected Python versions.
   lint-python:
     runs-on: ubuntu-latest
     container: "python:${{ matrix.python-version }}-slim"
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ '3.6.15', '3.10', '3.11' ]
+        python-version: [ '3.7', '3.10', '3.11' ]
 
     steps:
       - name: Checkout code
@@ -102,19 +102,19 @@ jobs:
       - name: Verify Python version
         run: python --version
 
-      - name: Verify requirements.txt dependencies can be installed
+      - name: Verify project dependencies can be installed
         shell: bash
         run: |
           cd paimon-python
           python -m pip install --upgrade pip
           TEMP_DIR=$(mktemp -d)
-          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || {
-            echo "ERROR: Failed to resolve dependencies from dev/requirements.txt"
+          python -m pip install . --target "$TEMP_DIR" || {
+            echo "ERROR: Failed to resolve project dependencies from pyproject.toml"
             rm -rf "$TEMP_DIR"
             exit 1
           }
           rm -rf "$TEMP_DIR"
-          echo "✓ dev/requirements.txt can be resolved on this Python"
+          echo "✓ project dependencies can be resolved on this Python"
 
       - name: Build Java
         run: |
@@ -125,10 +125,10 @@ jobs:
         shell: bash
         run: |
           df -h
-          if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then
-            python -m pip install --upgrade pip==21.3.1
+          if [[ "${{ matrix.python-version }}" == "3.7" ]]; then
+            python -m pip install --upgrade pip
             python --version
-            python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null
+            python -m pip install --no-cache-dir pyroaring==0.4.5 readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==5.3.3' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.3.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 packaging cramjam pyyaml flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 2>&1 >/dev/null
             python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/
           else
             python -m pip install --upgrade pip
@@ -142,7 +142,7 @@ jobs:
           df -h
 
       - name: Build and install tantivy-py from source
-        if: matrix.python-version != '3.6.15'
+        if: matrix.python-version != '3.7'
         shell: bash
         run: |
           pip install maturin[patchelf]
@@ -192,7 +192,7 @@ jobs:
           chmod +x paimon-python/dev/lint-python.sh
           ./paimon-python/dev/lint-python.sh -i pytest_torch
 
-  # One job: check dev/requirements.txt on each Python version in sequence, then Ray version test on 3.10.
+  # One job: check project dependencies on each Python version in sequence, then Ray version test on 3.10.
   requirement_version_compatible_test:
     runs-on: ubuntu-latest
     steps:
@@ -205,71 +205,71 @@ jobs:
             build-essential git curl cmake \
             && sudo rm -rf /var/lib/apt/lists/*
 
-      - name: Verify dev/requirements.txt on Python 3.8
+      - name: Verify project dependencies on Python 3.8
         uses: actions/setup-python@v5
         with:
           python-version: '3.8'
       - name: Resolve requirements (3.8)
         run: |
           cd paimon-python && python -m pip install --upgrade pip
           TEMP_DIR=$(mktemp -d)
-          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
-          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.8"
+          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
+          rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.8"
 
-      - name: Verify dev/requirements.txt on Python 3.9
+      - name: Verify project dependencies on Python 3.9
         uses: actions/setup-python@v5
         with:
           python-version: '3.9'
       - name: Resolve requirements (3.9)
         run: |
           cd paimon-python && python -m pip install --upgrade pip
           TEMP_DIR=$(mktemp -d)
-          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
-          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.9"
+          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
+          rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.9"
 
-      - name: Verify dev/requirements.txt on Python 3.10
+      - name: Verify project dependencies on Python 3.10
         uses: actions/setup-python@v5
         with:
           python-version: '3.10'
       - name: Resolve requirements (3.10)
         run: |
           cd paimon-python && python -m pip install --upgrade pip
           TEMP_DIR=$(mktemp -d)
-          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
-          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.10"
+          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
+          rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.10"
 
-      - name: Verify dev/requirements.txt on Python 3.11
+      - name: Verify project dependencies on Python 3.11
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
       - name: Resolve requirements (3.11)
         run: |
           cd paimon-python && python -m pip install --upgrade pip
           TEMP_DIR=$(mktemp -d)
-          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
-          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.11"
+          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
+          rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.11"
 
-      - name: Verify dev/requirements.txt on Python 3.12
+      - name: Verify project dependencies on Python 3.12
         uses: actions/setup-python@v5
         with:
           python-version: '3.12'
       - name: Resolve requirements (3.12)
         run: |
           cd paimon-python && python -m pip install --upgrade pip
           TEMP_DIR=$(mktemp -d)
-          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
-          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.12"
+          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
+          rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.12"
 
-      - name: Verify dev/requirements.txt on Python 3.13
+      - name: Verify project dependencies on Python 3.13
         uses: actions/setup-python@v5
         with:
           python-version: '3.13'
       - name: Resolve requirements (3.13)
         run: |
           cd paimon-python && python -m pip install --upgrade pip
           TEMP_DIR=$(mktemp -d)
-          python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
-          rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.13"
+          python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; }
+          rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.13"
 
       - name: Setup Python 3.10 for Ray test
         uses: actions/setup-python@v5

docs/docs/pypaimon/index.md

@@ -40,7 +40,7 @@ pip install pypaimon
 You can build the source package by executing the following command:
 
 ```commandline
-python3 setup.py sdist
+python3 -m build --sdist
 ```
 
 The package is under `dist/`. Then you can install the package by executing the following command:

paimon-python/MANIFEST.in

@@ -17,7 +17,6 @@
 ################################################################################
 include LICENSE
 include NOTICE
-include dev/requirements.txt
 include README.md
 recursive-include pypaimon *.py
 recursive-exclude pypaimon/tests *

paimon-python/README.md

@@ -8,19 +8,19 @@ This PyPi package contains the Python APIs for using Paimon.
 
 # Version
 
-Pypaimon requires Python 3.6+.
+Pypaimon requires Python 3.7+.
 
 # Dependencies
 
-The core dependencies are listed in `dev/requirements.txt`.
-The development dependencies are listed in `dev/requirements-dev.txt`.
+The core dependencies are listed in `pyproject.toml`.
+The development dependencies are listed in the `dev` dependency group in `pyproject.toml`.
 
 # Build
 
 You can build the source package by executing the following command:
 
 ```commandline
-python3 setup.py sdist
+python3 -m build --sdist
 ```
 
 The package is under `dist/`. Then you can install the package by executing the following command:
@@ -30,4 +30,3 @@ pip3 install dist/*.tar.gz
 ```
 
 The command will install the package and core dependencies to your local Python environment.
-

paimon-python/dev/check-licensing.sh

@@ -37,7 +37,7 @@ if [ -z "${SOURCE_PACKAGE}" ]; then
     cd ${PROJECT_ROOT}
 
     # Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root
-    if [ ! -f ${PROJECT_ROOT}/setup.py ]; then
+    if [ ! -f ${PROJECT_ROOT}/pyproject.toml ]; then
         echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory."
         exit 1
     fi

paimon-python/dev/lint-python.sh

@@ -196,9 +196,9 @@ function pytest_check() {
     echo "Detected Python version: $PYTHON_VERSION"
 
     # Determine test directory based on Python version
-    if [ "$PYTHON_VERSION" = "3.6" ]; then
+    if [ "$PYTHON_VERSION" = "3.6" ] || [ "$PYTHON_VERSION" = "3.7" ]; then
         TEST_DIR="pypaimon/tests/py36 pypaimon/tests/file_io_test.py"
-        echo "Running tests for Python 3.6: $TEST_DIR"
+        echo "Running tests for Python $PYTHON_VERSION: $TEST_DIR"
     else
         TEST_DIR="pypaimon/tests pypaimon/acceptance --ignore=pypaimon/tests/py36 --ignore=pypaimon/tests/e2e --ignore=pypaimon/tests/torch_read_test.py"
         echo "Running tests for Python $PYTHON_VERSION (excluding py36): pypaimon/tests pypaimon/acceptance --ignore=pypaimon/tests/py36"
@@ -359,4 +359,4 @@ done
 # collect checks according to the options
 collect_checks
 # run checks
-check_stage
+check_stage

paimon-python/pypaimon/filesystem/local_file_io.py

@@ -307,7 +307,7 @@ def write_parquet(self, path: str, data: pyarrow.Table, compression: str = 'zstd
     def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd',
                   zstd_level: int = 1, **kwargs):
         try:
-            import sys
+            import inspect
             import pyarrow.orc as orc
 
             file_path = self._to_file(path)
@@ -318,10 +318,10 @@ def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd',
             data = self._cast_time_columns_for_orc(data)
 
             with open(file_path, 'wb') as f:
-                if sys.version_info[:2] == (3, 6):
-                    orc.write_table(data, f, **kwargs)
-                else:
+                if 'compression' in inspect.signature(orc.write_table).parameters:
                     orc.write_table(data, f, compression=compression, **kwargs)
+                else:
+                    orc.write_table(data, f, **kwargs)
         except Exception as e:
             self.delete_quietly(path)
             raise RuntimeError(f"Failed to write ORC file {path}: {e}") from e

paimon-python/pypaimon/filesystem/pyarrow_file_io.py

@@ -567,23 +567,22 @@ def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd',
             (which is 3, see https://github.com/facebook/zstd/blob/dev/programs/zstdcli.c)
             instead of the specified level.
             """
-            import sys
+            import inspect
 
             import pyarrow.orc as orc
 
             data = self._cast_time_columns_for_orc(data)
 
             with self.new_output_stream(path) as output_stream:
-                # Check Python version - if 3.6, don't use compression parameter
-                if sys.version_info[:2] == (3, 6):
-                    orc.write_table(data, output_stream, **kwargs)
-                else:
+                if 'compression' in inspect.signature(orc.write_table).parameters:
                     orc.write_table(
                         data,
                         output_stream,
                         compression=compression,
                         **kwargs
                     )
+                else:
+                    orc.write_table(data, output_stream, **kwargs)
 
         except Exception as e:
             self.delete_quietly(path)