diff --git a/.github/workflows/paimon-python-checks.yml b/.github/workflows/paimon-python-checks.yml index 864d10eceecd..bfe99bb4d4e3 100755 --- a/.github/workflows/paimon-python-checks.yml +++ b/.github/workflows/paimon-python-checks.yml @@ -41,14 +41,14 @@ concurrency: cancel-in-progress: true jobs: - # Lint + test on 3.6 and 3.10 only (not every Python version). + # Lint + test on selected Python versions. lint-python: runs-on: ubuntu-latest container: "python:${{ matrix.python-version }}-slim" strategy: fail-fast: false matrix: - python-version: [ '3.6.15', '3.10', '3.11' ] + python-version: [ '3.7', '3.10', '3.11' ] steps: - name: Checkout code @@ -102,19 +102,19 @@ jobs: - name: Verify Python version run: python --version - - name: Verify requirements.txt dependencies can be installed + - name: Verify project dependencies can be installed shell: bash run: | cd paimon-python python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { - echo "ERROR: Failed to resolve dependencies from dev/requirements.txt" + python -m pip install . --target "$TEMP_DIR" || { + echo "ERROR: Failed to resolve project dependencies from pyproject.toml" rm -rf "$TEMP_DIR" exit 1 } rm -rf "$TEMP_DIR" - echo "✓ dev/requirements.txt can be resolved on this Python" + echo "✓ project dependencies can be resolved on this Python" - name: Build Java run: | @@ -125,10 +125,10 @@ jobs: shell: bash run: | df -h - if [[ "${{ matrix.python-version }}" == "3.6.15" ]]; then - python -m pip install --upgrade pip==21.3.1 + if [[ "${{ matrix.python-version }}" == "3.7" ]]; then + python -m pip install --upgrade pip python --version - python -m pip install --no-cache-dir pyroaring readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==4.2.4' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.1.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 dataclasses==0.8.0 flake8 pytest py4j==0.10.9.9 requests parameterized==0.8.1 2>&1 >/dev/null + python -m pip install --no-cache-dir pyroaring==0.4.5 readerwriterlock==1.0.9 'fsspec==2021.10.1' 'cachetools==5.3.3' 'ossfs==2021.8.0' pyarrow==6.0.1 pandas==1.3.5 'polars==0.9.12' 'fastavro==1.4.7' zstandard==0.19.0 packaging cramjam pyyaml flake8==4.0.1 pytest~=7.0 py4j==0.10.9.9 requests parameterized==0.9.0 2>&1 >/dev/null python -m pip install 'lumina-data>=${{ env.LUMINA_DATA_VERSION }}' -i https://pypi.org/simple/ else python -m pip install --upgrade pip @@ -142,7 +142,7 @@ jobs: df -h - name: Build and install tantivy-py from source - if: matrix.python-version != '3.6.15' + if: matrix.python-version != '3.7' shell: bash run: | pip install maturin[patchelf] @@ -192,7 +192,7 @@ jobs: chmod +x paimon-python/dev/lint-python.sh ./paimon-python/dev/lint-python.sh -i pytest_torch - # One job: check dev/requirements.txt on each Python version in sequence, then Ray version test on 3.10. + # One job: check project dependencies on each Python version in sequence, then Ray version test on 3.10. requirement_version_compatible_test: runs-on: ubuntu-latest steps: @@ -205,7 +205,7 @@ jobs: build-essential git curl cmake \ && sudo rm -rf /var/lib/apt/lists/* - - name: Verify dev/requirements.txt on Python 3.8 + - name: Verify project dependencies on Python 3.8 uses: actions/setup-python@v5 with: python-version: '3.8' @@ -213,10 +213,10 @@ jobs: run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.8" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.8" - - name: Verify dev/requirements.txt on Python 3.9 + - name: Verify project dependencies on Python 3.9 uses: actions/setup-python@v5 with: python-version: '3.9' @@ -224,10 +224,10 @@ jobs: run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.9" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.9" - - name: Verify dev/requirements.txt on Python 3.10 + - name: Verify project dependencies on Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' @@ -235,10 +235,10 @@ jobs: run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.10" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.10" - - name: Verify dev/requirements.txt on Python 3.11 + - name: Verify project dependencies on Python 3.11 uses: actions/setup-python@v5 with: python-version: '3.11' @@ -246,10 +246,10 @@ jobs: run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.11" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.11" - - name: Verify dev/requirements.txt on Python 3.12 + - name: Verify project dependencies on Python 3.12 uses: actions/setup-python@v5 with: python-version: '3.12' @@ -257,10 +257,10 @@ jobs: run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.12" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.12" - - name: Verify dev/requirements.txt on Python 3.13 + - name: Verify project dependencies on Python 3.13 uses: actions/setup-python@v5 with: python-version: '3.13' @@ -268,8 +268,8 @@ jobs: run: | cd paimon-python && python -m pip install --upgrade pip TEMP_DIR=$(mktemp -d) - python -m pip install -r dev/requirements.txt --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } - rm -rf "$TEMP_DIR" && echo "✓ dev/requirements.txt OK on Python 3.13" + python -m pip install . --target "$TEMP_DIR" || { rm -rf "$TEMP_DIR"; exit 1; } + rm -rf "$TEMP_DIR" && echo "✓ project dependencies OK on Python 3.13" - name: Setup Python 3.10 for Ray test uses: actions/setup-python@v5 diff --git a/docs/docs/pypaimon/index.md b/docs/docs/pypaimon/index.md index 18989aaf8289..ea0dd7ebf4da 100644 --- a/docs/docs/pypaimon/index.md +++ b/docs/docs/pypaimon/index.md @@ -40,7 +40,7 @@ pip install pypaimon You can build the source package by executing the following command: ```commandline -python3 setup.py sdist +python3 -m build --sdist ``` The package is under `dist/`. Then you can install the package by executing the following command: diff --git a/paimon-python/MANIFEST.in b/paimon-python/MANIFEST.in index 30776ec52c73..35d9eb3c445b 100644 --- a/paimon-python/MANIFEST.in +++ b/paimon-python/MANIFEST.in @@ -17,7 +17,6 @@ ################################################################################ include LICENSE include NOTICE -include dev/requirements.txt include README.md recursive-include pypaimon *.py recursive-exclude pypaimon/tests * diff --git a/paimon-python/README.md b/paimon-python/README.md index e216dc00197c..4ed71e32376d 100644 --- a/paimon-python/README.md +++ b/paimon-python/README.md @@ -8,19 +8,19 @@ This PyPi package contains the Python APIs for using Paimon. # Version -Pypaimon requires Python 3.6+. +Pypaimon requires Python 3.7+. # Dependencies -The core dependencies are listed in `dev/requirements.txt`. -The development dependencies are listed in `dev/requirements-dev.txt`. +The core dependencies are listed in `pyproject.toml`. +The development dependencies are listed in the `dev` dependency group in `pyproject.toml`. # Build You can build the source package by executing the following command: ```commandline -python3 setup.py sdist +python3 -m build --sdist ``` The package is under `dist/`. Then you can install the package by executing the following command: @@ -30,4 +30,3 @@ pip3 install dist/*.tar.gz ``` The command will install the package and core dependencies to your local Python environment. - diff --git a/paimon-python/dev/check-licensing.sh b/paimon-python/dev/check-licensing.sh index 883c43d5a852..46d0486241a4 100755 --- a/paimon-python/dev/check-licensing.sh +++ b/paimon-python/dev/check-licensing.sh @@ -37,7 +37,7 @@ if [ -z "${SOURCE_PACKAGE}" ]; then cd ${PROJECT_ROOT} # Sanity check to ensure that resolved paths are valid; a LICENSE file should always exist in project root - if [ ! -f ${PROJECT_ROOT}/setup.py ]; then + if [ ! -f ${PROJECT_ROOT}/pyproject.toml ]; then echo "Project root path ${PROJECT_ROOT} is not valid; script may be in the wrong directory." exit 1 fi diff --git a/paimon-python/dev/lint-python.sh b/paimon-python/dev/lint-python.sh index ffee46cbefa2..13c18fa5f48b 100755 --- a/paimon-python/dev/lint-python.sh +++ b/paimon-python/dev/lint-python.sh @@ -196,9 +196,9 @@ function pytest_check() { echo "Detected Python version: $PYTHON_VERSION" # Determine test directory based on Python version - if [ "$PYTHON_VERSION" = "3.6" ]; then + if [ "$PYTHON_VERSION" = "3.6" ] || [ "$PYTHON_VERSION" = "3.7" ]; then TEST_DIR="pypaimon/tests/py36 pypaimon/tests/file_io_test.py" - echo "Running tests for Python 3.6: $TEST_DIR" + echo "Running tests for Python $PYTHON_VERSION: $TEST_DIR" else TEST_DIR="pypaimon/tests pypaimon/acceptance --ignore=pypaimon/tests/py36 --ignore=pypaimon/tests/e2e --ignore=pypaimon/tests/torch_read_test.py" echo "Running tests for Python $PYTHON_VERSION (excluding py36): pypaimon/tests pypaimon/acceptance --ignore=pypaimon/tests/py36" @@ -359,4 +359,4 @@ done # collect checks according to the options collect_checks # run checks -check_stage \ No newline at end of file +check_stage diff --git a/paimon-python/dev/requirements-dev.txt b/paimon-python/dev/requirements-dev.txt deleted file mode 100644 index d4e9a0645b17..000000000000 --- a/paimon-python/dev/requirements-dev.txt +++ /dev/null @@ -1,31 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -# Core dependencies for pypaimon are in requirements.txt - -# Test dependencies for pypaimon are as follows -duckdb==1.3.2 -flake8==4.0.1 -pytest~=7.0 -# Ray: 2.48+ has no wheel for Python 3.8; use 2.10.0 on 3.8, 2.48.0 on 3.9+ -ray>=2.10.0 -requests -parameterized -# Vortex 0.71.0 regresses native predicate pushdown on single-row files. -vortex-data==0.70.0; python_version >= "3.11" -# Lumina vector search (optional, for lumina index tests) -lumina-data>=0.1.0 diff --git a/paimon-python/dev/requirements.txt b/paimon-python/dev/requirements.txt deleted file mode 100644 index dccbe98366a9..000000000000 --- a/paimon-python/dev/requirements.txt +++ /dev/null @@ -1,42 +0,0 @@ -################################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -################################################################################ -cachetools>=4.2,<6; python_version=="3.6" -cachetools>=5,<6; python_version>"3.6" -dataclasses>=0.8; python_version < "3.7" -fastavro>=1.4,<2 -fsspec>=2021.10,<2026; python_version<"3.8" -fsspec>=2023,<2026; python_version>="3.8" -packaging>=21,<26 -pandas>=1.1,<2; python_version < "3.7" -pandas>=1.3,<3; python_version >= "3.7" and python_version < "3.9" -pandas>=1.5,<3; python_version >= "3.9" -polars>=0.9,<1; python_version<"3.8" -polars>=1,<2; python_version>="3.8" -pyarrow>=6,<7; python_version < "3.8" -pyarrow>=16,<20; python_version >= "3.8" -pyroaring<=0.3.3; python_version < "3.7" -pyroaring<=0.4.5; python_version == "3.7" -pyroaring>=1.0.0; python_version >= "3.8" -readerwriterlock>=1,<2 -requests>=2.20,<3 -urllib3>=1.26,<3 -zstandard>=0.19,<1 -backports.zstd>=1.0.0,<1.4.0; python_version >= "3.9" and python_version < "3.14" -cramjam>=1.3.0,<3; python_version>="3.7" -pyyaml>=5.4,<7 -requests>=2.21.0,<3 diff --git a/paimon-python/pypaimon/filesystem/local_file_io.py b/paimon-python/pypaimon/filesystem/local_file_io.py index 8385790885c3..6a8d167162bd 100644 --- a/paimon-python/pypaimon/filesystem/local_file_io.py +++ b/paimon-python/pypaimon/filesystem/local_file_io.py @@ -307,7 +307,7 @@ def write_parquet(self, path: str, data: pyarrow.Table, compression: str = 'zstd def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd', zstd_level: int = 1, **kwargs): try: - import sys + import inspect import pyarrow.orc as orc file_path = self._to_file(path) @@ -318,10 +318,10 @@ def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd', data = self._cast_time_columns_for_orc(data) with open(file_path, 'wb') as f: - if sys.version_info[:2] == (3, 6): - orc.write_table(data, f, **kwargs) - else: + if 'compression' in inspect.signature(orc.write_table).parameters: orc.write_table(data, f, compression=compression, **kwargs) + else: + orc.write_table(data, f, **kwargs) except Exception as e: self.delete_quietly(path) raise RuntimeError(f"Failed to write ORC file {path}: {e}") from e diff --git a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py index 74d43579a28a..527d6bb0c5de 100644 --- a/paimon-python/pypaimon/filesystem/pyarrow_file_io.py +++ b/paimon-python/pypaimon/filesystem/pyarrow_file_io.py @@ -567,23 +567,22 @@ def write_orc(self, path: str, data: pyarrow.Table, compression: str = 'zstd', (which is 3, see https://github.com/facebook/zstd/blob/dev/programs/zstdcli.c) instead of the specified level. """ - import sys + import inspect import pyarrow.orc as orc data = self._cast_time_columns_for_orc(data) with self.new_output_stream(path) as output_stream: - # Check Python version - if 3.6, don't use compression parameter - if sys.version_info[:2] == (3, 6): - orc.write_table(data, output_stream, **kwargs) - else: + if 'compression' in inspect.signature(orc.write_table).parameters: orc.write_table( data, output_stream, compression=compression, **kwargs ) + else: + orc.write_table(data, output_stream, **kwargs) except Exception as e: self.delete_quietly(path) diff --git a/paimon-python/pyproject.toml b/paimon-python/pyproject.toml new file mode 100644 index 000000000000..6300d72b4736 --- /dev/null +++ b/paimon-python/pyproject.toml @@ -0,0 +1,129 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +[build-system] +requires = [ + "setuptools>=61", + "wheel", +] +build-backend = "setuptools.build_meta" + +[project] +name = "pypaimon" +version = "1.5.dev" +description = "Apache Paimon Python API" +readme = "README.md" +license = {text = "Apache-2.0"} +authors = [ + {name = "Apache Software Foundation", email = "dev@paimon.apache.org"}, +] +requires-python = ">=3.7" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "cachetools>=5,<6", + "fastavro>=1.4,<2", + 'fsspec>=2021.10,<2026; python_version<"3.8"', + 'fsspec>=2023,<2026; python_version>="3.8"', + "packaging>=21,<26", + 'pandas>=1.3,<3; python_version < "3.9"', + 'pandas>=1.5,<3; python_version >= "3.9"', + 'polars>=0.9,<1; python_version<"3.8"', + 'polars>=1,<2; python_version>="3.8"', + 'pyarrow>=6,<7; python_version < "3.8"', + 'pyarrow>=16,<20; python_version >= "3.8"', + 'pyroaring<=0.4.5; python_version == "3.7"', + 'pyroaring>=1.0.0; python_version >= "3.8"', + "readerwriterlock>=1,<2", + "urllib3>=1.26,<3", + "zstandard>=0.19,<1", + 'backports.zstd>=1.0.0,<1.4.0; python_version >= "3.9" and python_version < "3.14"', + "cramjam>=1.3.0,<3", + "pyyaml>=5.4,<7", + "requests>=2.21.0,<3", +] + +[project.optional-dependencies] +ray = [ + "ray>=2.10,<3", +] +torch = [ + "torch", +] +daft = [ + 'daft>=0.7.6; python_version>="3.10"', +] +oss = [ + 'ossfs>=2021.8; python_version<"3.8"', + 'ossfs>=2023; python_version>="3.8"', +] +jindo = [ + "pyjindosdk>=6.10.4", +] +lance = [ + 'pylance>=0.20,<1; python_version>="3.9"', + 'pylance>=0.10,<1; python_version>="3.8" and python_version<"3.9"', +] +vortex = [ + 'vortex-data==0.70.0; python_version>="3.11"', +] +lumina = [ + "lumina-data>=0.1.0", +] +sql = [ + 'pypaimon-rust; python_version>="3.10"', + 'datafusion>=52; python_version>="3.10"', +] + +[dependency-groups] +dev = [ + "duckdb==1.3.2", + "build>=1,<2", + "flake8==4.0.1", + "pytest~=7.0", + # Ray: 2.48+ has no wheel for Python 3.8; use 2.10.0 on 3.8, 2.48.0 on 3.9+ + "ray>=2.10.0", + "requests", + "parameterized", + # Vortex 0.71.0 regresses native predicate pushdown on single-row files. + 'vortex-data==0.70.0; python_version >= "3.11"', + # Lumina vector search (optional, for lumina index tests) + "lumina-data>=0.1.0", +] + +[project.scripts] +paimon = "pypaimon.cli:main" + +[project.urls] +Homepage = "https://paimon.apache.org" + +[tool.setuptools] +include-package-data = true + +[tool.setuptools.packages.find] +include = ["pypaimon*"] +exclude = ["pypaimon.tests*"] diff --git a/paimon-python/setup.py b/paimon-python/setup.py deleted file mode 100644 index aaba800fecb5..000000000000 --- a/paimon-python/setup.py +++ /dev/null @@ -1,202 +0,0 @@ -########################################################################## -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -########################################################################## -import atexit -import os -import shutil -import subprocess -import sys -import tarfile -import tempfile -from setuptools import find_packages, setup - -VERSION = "1.5.dev" - - -def get_dev_version(): - """Generate dev version with commit date. - Format: 1.5.devYYYYMMDD (e.g. 1.5.dev20260415) - Uses the commit date (author date) for reproducibility. - """ - base = VERSION.rstrip(".") - if not base.endswith("dev"): - return None - - try: - date_str = subprocess.check_output( - ["git", "log", "-1", "--format=%cd", "--date=format:%Y%m%d"], - stderr=subprocess.DEVNULL - ).decode("utf-8").strip() - except Exception: - print("Warning: git not available, skipping dev package.") - return None - - return base + date_str - - -def _build_dev_package(): - """After sdist completes, repack a copy with dev version.""" - if "sdist" not in sys.argv: - return - - dev_version = get_dev_version() - if dev_version is None: - return - - from packaging.version import Version - normalized = str(Version(VERSION)) - dev_normalized = str(Version(dev_version)) - - src_name = "pypaimon-{}".format(normalized) - dev_name = "pypaimon-{}".format(dev_normalized) - - src_tar = os.path.join("dist", src_name + ".tar.gz") - if not os.path.exists(src_tar): - return - - tmp_dir = tempfile.mkdtemp() - try: - with tarfile.open(src_tar, "r:gz") as tar: - tar.extractall(tmp_dir) - - src_dir = os.path.join(tmp_dir, src_name) - dev_dir = os.path.join(tmp_dir, dev_name) - os.rename(src_dir, dev_dir) - - # Update version in PKG-INFO files - for pkg_info in [ - os.path.join(dev_dir, "PKG-INFO"), - os.path.join(dev_dir, "pypaimon.egg-info", "PKG-INFO"), - ]: - if os.path.exists(pkg_info): - with open(pkg_info, "r") as f: - content = f.read() - content = content.replace( - "Version: " + normalized, - "Version: " + dev_normalized - ) - with open(pkg_info, "w") as f: - f.write(content) - - # Update VERSION in setup.py so pip install gets the correct version - setup_py = os.path.join(dev_dir, "setup.py") - if os.path.exists(setup_py): - with open(setup_py, "r") as f: - content = f.read() - content = content.replace( - 'VERSION = "' + VERSION + '"', - 'VERSION = "' + dev_version + '"' - ) - with open(setup_py, "w") as f: - f.write(content) - - dev_tar = os.path.join("dist", dev_name + ".tar.gz") - with tarfile.open(dev_tar, "w:gz") as tar: - tar.add(dev_dir, arcname=dev_name) - - print("Created dev package: " + dev_tar) - finally: - shutil.rmtree(tmp_dir) - - -atexit.register(_build_dev_package) - -PACKAGES = find_packages(include=["pypaimon*"], exclude=["pypaimon.tests*"]) - - -def read_requirements(): - """Read requirements from dev/requirements.txt file.""" - requirements_path = os.path.join(os.path.dirname(__file__), 'dev', 'requirements.txt') - requirements = [] - - if os.path.exists(requirements_path): - with open(requirements_path, 'r', encoding='utf-8') as f: - for line in f: - line = line.strip() - # Skip empty lines and comments - if line and not line.startswith('#'): - requirements.append(line) - - return requirements - - -install_requires = read_requirements() - -long_description = "See Apache Paimon Python API \ -[Doc](https://paimon.apache.org/docs/master/pypaimon/python-api/) for usage." - -setup( - name="pypaimon", - version=VERSION, - packages=PACKAGES, - include_package_data=True, - install_requires=install_requires, - entry_points={ - 'console_scripts': [ - 'paimon=pypaimon.cli:main', - ], - }, - extras_require={ - 'ray': [ - 'ray>=2.10,<3; python_version>="3.7"', - ], - 'torch': [ - 'torch', - ], - 'daft': [ - 'daft>=0.7.6; python_version>="3.10"', - ], - 'oss': [ - 'ossfs>=2021.8; python_version<"3.8"', - 'ossfs>=2023; python_version>="3.8"' - ], - 'jindo': [ - 'pyjindosdk>=6.10.4', - ], - 'lance': [ - 'pylance>=0.20,<1; python_version>="3.9"', - 'pylance>=0.10,<1; python_version>="3.8" and python_version<"3.9"' - ], - 'vortex': [ - 'vortex-data==0.70.0; python_version>="3.11"', - ], - 'lumina': [ - 'lumina-data>=0.1.0' - ], - 'sql': [ - 'pypaimon-rust; python_version>="3.10"', - 'datafusion>=52; python_version>="3.10"', - ], - }, - description="Apache Paimon Python API", - long_description=long_description, - long_description_content_type="text/markdown", - author="Apache Software Foundation", - author_email="dev@paimon.apache.org", - url="https://paimon.apache.org", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - ], - python_requires=">=3.6", -)