From e8fe520d35c457cedbb81587b5535ddb448bc26c Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 11 Mar 2026 22:07:36 +0800 Subject: [PATCH 1/3] support release python by ci. --- .github/workflows/unit-test-cpp.yml | 6 + .github/workflows/unit-test-python.yml | 33 ++ .github/workflows/wheels.yml | 286 ++++++++++++++++++ .gitignore | 2 +- cpp/CMakeLists.txt | 2 +- cpp/pom.xml | 10 - cpp/src/CMakeLists.txt | 2 + cpp/third_party/zlib-1.3.1/treebuild.xml | 116 ------- .../zlib-1.3.1/zlib-1.3.1/treebuild.xml | 116 ------- pom.xml | 10 +- python/VersionUpdater.groovy | 20 +- python/pyproject.toml | 69 +++++ python/requirements.txt | 2 +- python/setup.py | 231 +++++++------- python/tsfile/__init__.py | 13 +- python/tsfile/tsfile_cpp.pxd | 6 +- 16 files changed, 545 insertions(+), 379 deletions(-) create mode 100644 .github/workflows/wheels.yml delete mode 100644 cpp/third_party/zlib-1.3.1/treebuild.xml delete mode 100644 cpp/third_party/zlib-1.3.1/zlib-1.3.1/treebuild.xml create mode 100644 python/pyproject.toml diff --git a/.github/workflows/unit-test-cpp.yml b/.github/workflows/unit-test-cpp.yml index 1c3495ea6..2fab96567 100644 --- a/.github/workflows/unit-test-cpp.yml +++ b/.github/workflows/unit-test-cpp.yml @@ -109,6 +109,12 @@ jobs: shell: bash run: | if [[ "$RUNNER_OS" == "Linux" ]]; then + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y uuid-dev + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libuuid-devel + fi sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-17 100 sudo update-alternatives --set clang-format /usr/bin/clang-format-17 sudo apt-get update diff --git a/.github/workflows/unit-test-python.yml b/.github/workflows/unit-test-python.yml index 708f8ec35..98fdc2f15 100644 --- a/.github/workflows/unit-test-python.yml +++ b/.github/workflows/unit-test-python.yml @@ -51,6 +51,11 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + # Setup caching of the artifacts in the .m2 directory, so they don't have to # all be downloaded again for every build. - name: Cache Maven packages @@ -60,6 +65,17 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2- + - name: Install dependencies + shell: bash + run: | + if [[ "$RUNNER_OS" == "Linux" ]]; then + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + sudo apt-get install -y uuid-dev + elif command -v yum >/dev/null 2>&1; then + sudo yum install -y libuuid-devel + fi + fi # On Windows systems the 'mvnw' script needs an additional ".cmd" appended. - name: Calculate platform suffix id: platform_suffix @@ -103,6 +119,23 @@ jobs: fi ./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-python -Denable.asan=OFF -Dbuild.type=Release clean verify -Dspotless.skip=true + # Debug: show DLL dependencies on Windows to diagnose load failures + - name: Debug DLL dependencies + if: runner.os == 'Windows' && failure() + shell: bash + run: | + echo "=== Files in python/tsfile/ ===" + ls -la python/tsfile/*.dll python/tsfile/*.pyd 2>/dev/null || echo "No .dll/.pyd found" + echo "" + echo "=== libtsfile.dll dependencies ===" + objdump -p python/tsfile/libtsfile.dll 2>/dev/null | grep "DLL Name" || echo "objdump failed" + echo "" + echo "=== tsfile_reader.pyd dependencies ===" + objdump -p python/tsfile/tsfile_reader*.pyd 2>/dev/null | grep "DLL Name" || echo "objdump failed" + echo "" + echo "=== tsfile_writer.pyd dependencies ===" + objdump -p python/tsfile/tsfile_writer*.pyd 2>/dev/null | grep "DLL Name" || echo "objdump failed" + - name: Upload whl Artifact uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..bfcb7f1ed --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,286 @@ +name: Build TsFile wheels(multi-platform) + +on: + push: + branches: + - "rc/**" + workflow_dispatch: + +jobs: + build: + name: Build wheels on ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - name: linux-x86_64 + os: ubuntu-22.04 + platform: linux + cibw_archs_linux: "x86_64" + + - name: linux-aarch64 + os: ubuntu-22.04-arm + platform: linux + cibw_archs_linux: "aarch64" + + - name: macos-x86_64 + os: macos-15-intel + platform: macos + cibw_archs_macos: "x86_64" + + - name: macos-arm64 + os: macos-latest + platform: macos + cibw_archs_macos: "arm64" +# Windows is handled by the build-windows job below + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Install system deps (macOS) + if: matrix.platform == 'macos' + run: | + set -eux + brew update + brew install pkg-config || true + + - name: Install build tools + run: | + python -m pip install -U pip wheel + python -m pip install cibuildwheel==2.21.3 + + - name: Pre-download virtualenv for cibuildwheel + run: | + if [ "$(uname)" = "Darwin" ]; then + CACHE_DIR="$HOME/Library/Caches/cibuildwheel" + else + CACHE_DIR="$HOME/.cache/cibuildwheel" + fi + mkdir -p "$CACHE_DIR" + TARGET="$CACHE_DIR/virtualenv-20.26.6.pyz" + if [ ! -f "$TARGET" ]; then + curl -sSL --retry 5 --retry-delay 15 --retry-all-errors \ + -o "$TARGET" \ + "https://github.com/pypa/get-virtualenv/raw/20.26.6/public/virtualenv.pyz" + fi + ls -la "$TARGET" + + + - name: Build C++ core via Maven (macOS) + if: matrix.platform == 'macos' + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: "12.0" + CFLAGS: "-mmacosx-version-min=12.0" + CXXFLAGS: "-mmacosx-version-min=12.0" + LDFLAGS: "-mmacosx-version-min=12.0" + run: | + set -euxo pipefail + chmod +x mvnw || true + ./mvnw -Pwith-cpp clean package \ + -DskipTests -Dspotless.check.skip=true -Dspotless.apply.skip=true \ + -Dbuild.test=OFF \ + -Dcmake.args="-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0" + otool -l cpp/target/build/lib/libtsfile*.dylib | grep -A2 LC_VERSION_MIN_MACOSX || true + + - name: Build wheels via cibuildwheel + if: matrix.platform != 'macos' + env: + CIBW_ARCHS_LINUX: ${{ matrix.cibw_archs_linux }} +# CIBW_ARCHS_WINDOWS: ${{ matrix.cibw_archs_windows }} + + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" + CIBW_SKIP: "pp* *-musllinux*" + + CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2014" + CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux2014" + + MACOSX_DEPLOYMENT_TARGET: "12.0" + + CIBW_BEFORE_ALL_LINUX: | + set -euxo pipefail + if command -v yum >/dev/null 2>&1; then + yum install -y wget tar gzip pkgconfig libuuid-devel libblkid-devel + else + echo "Not a yum-based image?" ; exit 1 + fi + ARCH="$(uname -m)" + mkdir -p /opt/java + if [ "$ARCH" = "x86_64" ]; then + JDK_URL="https://download.oracle.com/java/17/archive/jdk-17.0.12_linux-x64_bin.tar.gz" + else + # aarch64 + JDK_URL="https://download.oracle.com/java/17/archive/jdk-17.0.12_linux-aarch64_bin.tar.gz" + fi + curl -L -o /tmp/jdk17.tar.gz "$JDK_URL" + tar -xzf /tmp/jdk17.tar.gz -C /opt/java + export JAVA_HOME=$(echo /opt/java/jdk-17.0.12*) + export PATH="$JAVA_HOME/bin:$PATH" + java -version + + chmod +x mvnw || true + ./mvnw -Pwith-cpp clean package \ + -DskipTests -Dbuild.test=OFF \ + -Dspotless.check.skip=true -Dspotless.apply.skip=true + test -d cpp/target/build/lib && test -d cpp/target/build/include + + CIBW_TEST_COMMAND: > + python -c "import tsfile, tsfile.tsfile_reader as r; print('import-ok:')" + CIBW_BUILD_VERBOSITY: "1" + run: cibuildwheel --output-dir wheelhouse python + + - name: Build wheels via cibuildwheel (macOS) + if: matrix.platform == 'macos' + env: + CIBW_ARCHS_MACOS: ${{ matrix.cibw_archs_macos }} + CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" +# CIBW_BUILD: "cp313-*" + CIBW_SKIP: "pp*" + CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=12.0" + MACOSX_DEPLOYMENT_TARGET: "12.0" + CIBW_TEST_COMMAND: > + python -c "import tsfile, tsfile.tsfile_reader as r; print('import-ok:')" + CIBW_BUILD_VERBOSITY: "1" + run: cibuildwheel --output-dir wheelhouse python + + - name: Upload wheels as artifact + uses: actions/upload-artifact@v4 + with: + name: tsfile-wheels-${{ matrix.name }} + path: wheelhouse/*.whl + + # ── Windows: build C++ once, then build wheels for each Python version ── + build-windows-cpp: + name: Build C++ core (Windows) + runs-on: windows-2022 + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + fetch-depth: 0 + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up MSYS2 / MinGW + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + mingw-w64-x86_64-gcc + mingw-w64-x86_64-cmake + mingw-w64-x86_64-make + make + + - name: Build C++ core via Maven + shell: msys2 {0} + run: | + set -euxo pipefail + export JAVA_HOME="$(cygpath "$JAVA_HOME")" + export PATH="$JAVA_HOME/bin:$PATH" + java -version + chmod +x mvnw || true + ./mvnw -Pwith-cpp clean package \ + -DskipTests -Dbuild.test=OFF \ + -Dspotless.check.skip=true -Dspotless.apply.skip=true + test -d cpp/target/build/lib + test -d cpp/target/build/include + + - name: Upload C++ build output + uses: actions/upload-artifact@v4 + with: + name: tsfile-cpp-windows + path: | + cpp/target/build/lib/ + cpp/target/build/include/ + + build-windows-wheels: + name: Build wheel (Windows, Python ${{ matrix.python-version }}) + needs: build-windows-cpp + runs-on: windows-2022 + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: false + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Set up Java 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: "17" + + - name: Set up MSYS2 / MinGW + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: false + install: >- + mingw-w64-x86_64-gcc + mingw-w64-x86_64-make + make + + - name: Download C++ build output + uses: actions/download-artifact@v4 + with: + name: tsfile-cpp-windows + path: cpp/target/build/ + + - name: Build wheel + shell: msys2 {0} + run: | + set -euxo pipefail + export JAVA_HOME="$(cygpath "$JAVA_HOME")" + export PYTHON_HOME="$(cygpath "$pythonLocation")" + export PATH="$PYTHON_HOME:$PYTHON_HOME/Scripts:$JAVA_HOME/bin:$PATH" + + # Build wheel via Maven (no clean — keep C++ artifacts from previous job) + chmod +x mvnw || true + cd python + ../mvnw package -DskipTests \ + -Dspotless.check.skip=true -Dspotless.apply.skip=true + ls -la dist/ + + - name: Verify wheel + shell: bash + run: | + python -m pip install python/dist/*.whl + python -c "import tsfile, tsfile.tsfile_reader as r; print('import-ok')" + + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: tsfile-wheels-windows-py${{ matrix.python-version }} + path: python/dist/*.whl + diff --git a/.gitignore b/.gitignore index 4fa45012a..8f2e34e3f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ python/tsfile/include cpp/cmake-build-debug-mingw/ cpp/third_party/googletest-release-1.12.1.zip -cpp/third_party/zlib-1.2.13/zconf.h +cpp/third_party/zlib-1.3.1 .vscode/ build/* diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c85150d8f..244faef2f 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -1,4 +1,4 @@ -#[[ +#[[ Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information diff --git a/cpp/pom.xml b/cpp/pom.xml index b31b2dc12..b1836e309 100644 --- a/cpp/pom.xml +++ b/cpp/pom.xml @@ -163,16 +163,6 @@ - - linux-install-uuid-dev - - - unix - Linux - - - - jenkins-build diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index e922836b7..6ed41a423 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -134,6 +134,8 @@ target_link_libraries(write_obj ${COMPRESSION_LIBS}) add_library(tsfile SHARED) + + if (${COV_ENABLED}) message("Enable code cov...") if (ENABLE_ANTLR4) diff --git a/cpp/third_party/zlib-1.3.1/treebuild.xml b/cpp/third_party/zlib-1.3.1/treebuild.xml deleted file mode 100644 index 930b00be4..000000000 --- a/cpp/third_party/zlib-1.3.1/treebuild.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - zip compression library - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/cpp/third_party/zlib-1.3.1/zlib-1.3.1/treebuild.xml b/cpp/third_party/zlib-1.3.1/zlib-1.3.1/treebuild.xml deleted file mode 100644 index 930b00be4..000000000 --- a/cpp/third_party/zlib-1.3.1/zlib-1.3.1/treebuild.xml +++ /dev/null @@ -1,116 +0,0 @@ - - - - zip compression library - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/pom.xml b/pom.xml index cb06548b4..561e95356 100644 --- a/pom.xml +++ b/pom.xml @@ -135,6 +135,10 @@ **/tsfile.egg-info/** **/third_party/** + **/.python-version + **/**venv-py**/** + **/.python-version + python/.python-version @@ -445,6 +449,8 @@ 4 **/target/** + python/.python-version + **/.python-version @@ -751,8 +757,8 @@ win windows-amd64 MinGW Makefiles - venv/Scripts/ - python + + python.exe diff --git a/python/VersionUpdater.groovy b/python/VersionUpdater.groovy index ee3eab2b8..a586fe936 100644 --- a/python/VersionUpdater.groovy +++ b/python/VersionUpdater.groovy @@ -21,13 +21,15 @@ // Synchronize the version in setup.py and the one used in the maven pom. //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -def pyProjectFile = new File(project.basedir, "setup.py") def currentMavenVersion = project.version as String def currentPyVersion = currentMavenVersion if(currentMavenVersion.contains("-SNAPSHOT")) { currentPyVersion = currentMavenVersion.split("-SNAPSHOT")[0] + ".dev" } println "Current Project Version in Maven: " + currentMavenVersion + +// Sync setup.py +def pyProjectFile = new File(project.basedir, "setup.py") def match = pyProjectFile.text =~ /version\s*=\s*"(.*?)"/ def pyProjectFileVersion = match[0][1] println "Current Project Version in setup.py: " + pyProjectFileVersion @@ -35,7 +37,21 @@ println "Current Project Version in setup.py: " + pyProjectFileVersion if (pyProjectFileVersion != currentPyVersion) { pyProjectFile.text = pyProjectFile.text.replace("version = \"" + pyProjectFileVersion + "\"", "version = \"" + currentPyVersion + "\"") println "Version in setup.py updated from " + pyProjectFileVersion + " to " + currentPyVersion - // TODO: When releasing, we might need to manually add this file to the release preparation commit. } else { println "Version in setup.py is up to date" } + +// Sync pyproject.toml +def pyprojectTomlFile = new File(project.basedir, "pyproject.toml") +if (pyprojectTomlFile.exists()) { + def tomlMatch = pyprojectTomlFile.text =~ /version\s*=\s*"(.*?)"/ + def tomlVersion = tomlMatch[0][1] + println "Current Project Version in pyproject.toml: " + tomlVersion + + if (tomlVersion != currentPyVersion) { + pyprojectTomlFile.text = pyprojectTomlFile.text.replaceFirst("version = \"" + tomlVersion + "\"", "version = \"" + currentPyVersion + "\"") + println "Version in pyproject.toml updated from " + tomlVersion + " to " + currentPyVersion + } else { + println "Version in pyproject.toml is up to date" + } +} diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 000000000..86229aa6d --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +[build-system] +requires = [ + "setuptools>=69", + "wheel", + "Cython>=3", + "numpy>=2.0.0,<3" + ] + +build-backend = "setuptools.build_meta" + +[project] +name = "tsfile" +version = "2.2.1.dev" +requires-python = ">=3.9" +description = "TsFile Python" +readme = {file = "README.md", content-type = "text/markdown"} +maintainers = [ + {name = "Apache TsFile Developers", email = "dev@tsfile.apache.org"} +] +dependencies = [ + "numpy>=2.0.0,<3", + "pandas>=2.0" +] + +[project.urls] +Homepage = "https://tsfile.apache.org/" +Documentation = "https://tsfile.apache.org/zh/UserGuide/latest/QuickStart/Navigating_Time_Series_Data.html" +Repository = "https://github.com/apache/tsfile" +Issues = "https://github.com/apache/tsfile/issues" + +[tool.setuptools] +package-dir = {"" = "."} + +[tool.setuptools.packages.find] +where = ["."] +include = ["tsfile*"] + +[tool.setuptools.package-data] +tsfile = [ + "*.pxd", + "*.pxi", + "*.so", + "*.so.*", + "*.dylib", + "*.dylib.*", + "*.dll", + "*.dll.a", + "*.lib", + "*.lib.a", + "include/**/*" +] diff --git a/python/requirements.txt b/python/requirements.txt index 5b1c19aa2..bcccac286 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -18,7 +18,7 @@ # cython==3.0.10 -numpy==1.26.4 +numpy>=2.0.0,<3 pandas==2.2.2 setuptools==78.1.1 wheel==0.46.2 diff --git a/python/setup.py b/python/setup.py index 9f2a1a37a..db95ae6ae 100644 --- a/python/setup.py +++ b/python/setup.py @@ -19,151 +19,134 @@ import os import platform import shutil - +import sys import numpy as np + +from pathlib import Path from Cython.Build import cythonize from setuptools import setup, Extension from setuptools.command.build_ext import build_ext +ROOT = Path(__file__).parent.resolve() +PKG = ROOT / "tsfile" +CPP_OUT = ROOT / ".." / "cpp" / "target" / "build" +CPP_LIB = CPP_OUT / "lib" +CPP_INC = CPP_OUT / "include" + version = "2.2.1.dev" system = platform.system() - -def copy_tsfile_lib(source_dir, target_dir, suffix): - lib_file_name = f"libtsfile.{suffix}" - source = os.path.join(source_dir, lib_file_name) - target = os.path.join(target_dir, lib_file_name) - - if os.path.exists(source): - shutil.copyfile(source, target) - - if system == "Linux": - link_name = os.path.join(target_dir, "libtsfile.so") - if os.path.exists(link_name): - os.remove(link_name) - os.symlink(lib_file_name, link_name) - elif system == "Darwin": - link_name = os.path.join(target_dir, "libtsfile.dylib") - if os.path.exists(link_name): - os.remove(link_name) - os.symlink(lib_file_name, link_name) - - -project_dir = os.path.dirname(os.path.abspath(__file__)) -tsfile_py_include = os.path.join(project_dir, "tsfile", "include") - -if os.path.exists(tsfile_py_include): - shutil.rmtree(tsfile_py_include) - -shutil.copytree( - os.path.join(project_dir, "..", "cpp", "target", "build", "include"), - os.path.join(tsfile_py_include, ""), -) - - -def copy_tsfile_header(source): - for file in source: - if os.path.exists(file): - target = os.path.join(tsfile_py_include, os.path.basename(file)) - shutil.copyfile(file, target) - - -## Copy C wrapper header. -# tsfile/cpp/src/cwrapper/tsfile_cwrapper.h -source_headers = [ - os.path.join(project_dir, "..", "cpp", "src", "cwrapper", "tsfile_cwrapper.h"), -] - -copy_tsfile_header(source_headers) - -## Copy shared library -tsfile_shared_source_dir = os.path.join(project_dir, "..", "cpp", "target", "build", "lib") -tsfile_shared_dir = os.path.join(project_dir, "tsfile") - -if system == "Darwin": - copy_tsfile_lib(tsfile_shared_source_dir, tsfile_shared_dir, version + ".dylib") -elif system == "Linux": - copy_tsfile_lib(tsfile_shared_source_dir, tsfile_shared_dir, "so." + version) +(PKG / "include").mkdir(exist_ok=True) +if (PKG / "include").exists() and CPP_INC.exists(): + shutil.rmtree(PKG / "include") + shutil.copytree(CPP_INC, PKG / "include") +if sys.platform.startswith("linux"): + candidates = sorted(CPP_LIB.glob("libtsfile.so*"), key=lambda p: len(p.name), reverse=True) + if not candidates: + raise FileNotFoundError("missing libtsfile.so* in build output") + src = candidates[0] + dst = PKG / src.name + shutil.copy2(src, dst) + link_name = PKG / "libtsfile.so" + shutil.copy2(src, link_name) + +elif sys.platform == "darwin": + candidates = sorted(CPP_LIB.glob("libtsfile.*.dylib")) or list(CPP_LIB.glob("libtsfile.dylib")) + if not candidates: + raise FileNotFoundError("missing libtsfile*.dylib in build output") + src = candidates[0] + dst = PKG / src.name + shutil.copy2(src, dst) + link_name = PKG / "libtsfile.dylib" + shutil.copy2(src, link_name) +elif sys.platform == "win32": + for base_name in ("libtsfile",): + dll_candidates = sorted(CPP_LIB.glob(f"{base_name}*.dll"), key=lambda p: len(p.name), reverse=True) + dll_a_candidates = sorted(CPP_LIB.glob(f"{base_name}*.dll.a"), key=lambda p: len(p.name), reverse=True) + + if not dll_candidates: + raise FileNotFoundError(f"missing {base_name}*.dll in build output") + if not dll_a_candidates: + raise FileNotFoundError(f"missing {base_name}*.dll.a in build output") + + dll_src = dll_candidates[0] + dll_a_src = dll_a_candidates[0] + + shutil.copy2(dll_src, PKG / f"{base_name}.dll") + shutil.copy2(dll_a_src, PKG / f"{base_name}.dll.a") + + # Copy MinGW runtime DLLs next to libtsfile.dll so Python can find them. + # Python 3.8+ does not search PATH for DLLs; they must be in the same + # directory as the .pyd extensions (registered via os.add_dll_directory). + for _mingw_dll in ("libstdc++-6.dll", "libgcc_s_seh-1.dll", "libwinpthread-1.dll"): + for _dir in os.environ.get("PATH", "").split(os.pathsep): + _src = Path(_dir) / _mingw_dll + if _src.is_file(): + shutil.copy2(_src, PKG / _mingw_dll) + print(f"setup.py: copied {_mingw_dll} from {_src}") + break + else: + print(f"setup.py: WARNING - {_mingw_dll} not found on PATH") else: - copy_tsfile_lib(tsfile_shared_source_dir, tsfile_shared_dir, "dll") - -tsfile_include_dir = os.path.join(project_dir, "tsfile", "include") - -ext_modules_tsfile = [ - # utils: from python to c or c to python. - Extension( - "tsfile.tsfile_py_cpp", - sources=[os.path.join("tsfile", "tsfile_py_cpp.pyx")], - libraries=["tsfile"], - library_dirs=[tsfile_shared_dir], - include_dirs=[tsfile_include_dir, np.get_include()], - runtime_library_dirs=[tsfile_shared_dir] if system != "Windows" else None, - extra_compile_args=( - ["-std=c++11"] if system != "Windows" else ["-std=c++11", "-DMS_WIN64"] - ), - language="c++", - ), - # query data and describe schema: tsfile reader module - Extension( - "tsfile.tsfile_reader", - sources=[os.path.join("tsfile", "tsfile_reader.pyx")], - libraries=["tsfile"], - library_dirs=[tsfile_shared_dir], - depends=[os.path.join("tsfile", "tsfile_py_cpp.pxd")], - include_dirs=[tsfile_include_dir, np.get_include()], - runtime_library_dirs=[tsfile_shared_dir] if system != "Windows" else None, - extra_compile_args=( - ["-std=c++11"] if system != "Windows" else ["-std=c++11", "-DMS_WIN64"] - ), - language="c++", - ), - # write data and register schema: tsfile writer module - Extension( - "tsfile.tsfile_writer", - sources=[os.path.join("tsfile", "tsfile_writer.pyx")], - libraries=["tsfile"], - library_dirs=[tsfile_shared_dir], - depends=[os.path.join("tsfile", "tsfile_py_cpp.pxd")], - include_dirs=[tsfile_include_dir, np.get_include()], - runtime_library_dirs=[tsfile_shared_dir] if system != "Windows" else None, - extra_compile_args=( - ["-std=c++11"] if system != "Windows" else ["-std=c++11", "-DMS_WIN64"] - ), - language="c++", - ) -] + raise RuntimeError(f"Unsupported platform: {sys.platform}") class BuildExt(build_ext): - def build_extensions(self): - numpy_include = np.get_include() - for ext in self.extensions: - ext.include_dirs.append(numpy_include) - super().build_extensions() + def run(self): + super().run() def finalize_options(self): - if system == "Windows": + if sys.platform == "win32": self.compiler = "mingw32" super().finalize_options() +extra_compile_args = [] +extra_link_args = [] +runtime_library_dirs = [] +libraries = [] +library_dirs = [str(PKG)] +include_dirs = [str(PKG), np.get_include(), str(PKG / "include")] + +if sys.platform.startswith("linux"): + libraries = ["tsfile"] + extra_compile_args += ["-O3", "-std=c++11", "-fvisibility=hidden", "-fPIC"] + runtime_library_dirs = ["$ORIGIN"] + extra_link_args += ["-Wl,-rpath,$ORIGIN"] +elif sys.platform == "darwin": + libraries = ["tsfile"] + extra_compile_args += ["-O3", "-std=c++11", "-fvisibility=hidden", "-fPIC"] + extra_link_args += ["-Wl,-rpath,@loader_path", "-stdlib=libc++"] +elif sys.platform == "win32": + libraries = ["Tsfile"] + extra_compile_args += ["-O2", "-std=c++11", "-DSIZEOF_VOID_P=8", "-D__USE_MINGW_ANSI_STDIO=1", "-DMS_WIN64", + "-D_WIN64"] + extra_link_args += [] +else: + raise RuntimeError(f"Unsupported platform: {sys.platform}") + +common = dict( + language="c++", + include_dirs=include_dirs, + library_dirs=library_dirs, + libraries=libraries, + extra_compile_args=extra_compile_args, + extra_link_args=extra_link_args, + runtime_library_dirs=runtime_library_dirs, +) + +exts = [ + Extension("tsfile.tsfile_py_cpp", ["tsfile/tsfile_py_cpp.pyx"], **common), + Extension("tsfile.tsfile_reader", ["tsfile/tsfile_reader.pyx"], **common), + Extension("tsfile.tsfile_writer", ["tsfile/tsfile_writer.pyx"], **common), +] + setup( name="tsfile", version=version, - description="Tsfile reader and writer for python", - url="https://tsfile.apache.org", - author='"Apache TsFile"', packages=["tsfile"], - license="Apache 2.0", - ext_modules=cythonize(ext_modules_tsfile), - cmdclass={"build_ext": BuildExt}, - include_dirs=[np.get_include()], - package_dir={"tsfile": "./tsfile"}, - package_data={ - "tsfile": [ - "libtsfile.*", - "*.pxd" - ] - }, + package_dir={"": "."}, include_package_data=True, + ext_modules=cythonize(exts, compiler_directives={"language_level": 3}), + cmdclass={"build_ext": BuildExt}, ) diff --git a/python/tsfile/__init__.py b/python/tsfile/__init__.py index a9237257b..7954a216d 100644 --- a/python/tsfile/__init__.py +++ b/python/tsfile/__init__.py @@ -19,9 +19,16 @@ import ctypes import os import platform -system = platform.system() -if system == "Windows": - ctypes.WinDLL(os.path.join(os.path.dirname(__file__), "libtsfile.dll"), winmode=0) +import sys + +if sys.platform == "win32": + _pkg_dir = os.path.dirname(os.path.abspath(__file__)) + os.add_dll_directory(_pkg_dir) + # Preload libtsfile.dll with absolute path to bypass DLL search issues. + # This ensures it's already in memory when .pyd extensions reference it. + _tsfile_dll = os.path.join(_pkg_dir, "libtsfile.dll") + if os.path.isfile(_tsfile_dll): + ctypes.CDLL(_tsfile_dll) from .constants import * from .schema import * diff --git a/python/tsfile/tsfile_cpp.pxd b/python/tsfile/tsfile_cpp.pxd index 9c65fb26f..f90b23089 100644 --- a/python/tsfile/tsfile_cpp.pxd +++ b/python/tsfile/tsfile_cpp.pxd @@ -22,7 +22,7 @@ from libc.stdint cimport uint32_t, int32_t, int64_t, uint64_t, uint8_t ctypedef int32_t ErrorCode # import symbols from tsfile_cwrapper.h -cdef extern from "./tsfile_cwrapper.h": +cdef extern from "cwrapper/tsfile_cwrapper.h": # common ctypedef int64_t timestamp @@ -215,7 +215,7 @@ cdef extern from "./tsfile_cwrapper.h": -cdef extern from "./common/config/config.h" namespace "common": +cdef extern from "common/config/config.h" namespace "common": cdef cppclass ConfigValue: uint32_t tsblock_mem_inc_step_size_ uint32_t tsblock_max_memory_ @@ -237,7 +237,7 @@ cdef extern from "./common/config/config.h" namespace "common": uint8_t string_encoding_type_; uint8_t default_compression_type_; -cdef extern from "./common/global.h" namespace "common": +cdef extern from "common/global.h" namespace "common": ConfigValue g_config_value_ int set_datatype_encoding(uint8_t data_type, uint8_t encoding) int set_global_compression(uint8_t compression) From a5007d1460e3380cd4460f420bf2db6fdb7fe675 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Sat, 14 Mar 2026 16:30:27 +0800 Subject: [PATCH 2/3] fix format. --- .github/workflows/unit-test-python.yml | 17 ----------------- cpp/src/CMakeLists.txt | 2 -- 2 files changed, 19 deletions(-) diff --git a/.github/workflows/unit-test-python.yml b/.github/workflows/unit-test-python.yml index 98fdc2f15..f99eb3b4b 100644 --- a/.github/workflows/unit-test-python.yml +++ b/.github/workflows/unit-test-python.yml @@ -119,23 +119,6 @@ jobs: fi ./mvnw${{ steps.platform_suffix.outputs.platform_suffix }} -P with-python -Denable.asan=OFF -Dbuild.type=Release clean verify -Dspotless.skip=true - # Debug: show DLL dependencies on Windows to diagnose load failures - - name: Debug DLL dependencies - if: runner.os == 'Windows' && failure() - shell: bash - run: | - echo "=== Files in python/tsfile/ ===" - ls -la python/tsfile/*.dll python/tsfile/*.pyd 2>/dev/null || echo "No .dll/.pyd found" - echo "" - echo "=== libtsfile.dll dependencies ===" - objdump -p python/tsfile/libtsfile.dll 2>/dev/null | grep "DLL Name" || echo "objdump failed" - echo "" - echo "=== tsfile_reader.pyd dependencies ===" - objdump -p python/tsfile/tsfile_reader*.pyd 2>/dev/null | grep "DLL Name" || echo "objdump failed" - echo "" - echo "=== tsfile_writer.pyd dependencies ===" - objdump -p python/tsfile/tsfile_writer*.pyd 2>/dev/null | grep "DLL Name" || echo "objdump failed" - - name: Upload whl Artifact uses: actions/upload-artifact@v7 with: diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 6ed41a423..e922836b7 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -134,8 +134,6 @@ target_link_libraries(write_obj ${COMPRESSION_LIBS}) add_library(tsfile SHARED) - - if (${COV_ENABLED}) message("Enable code cov...") if (ENABLE_ANTLR4) From a6288f83df528ef2ab79ac8a347479ccbc47ac34 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 19 Mar 2026 17:29:26 +0800 Subject: [PATCH 3/3] fix comment. --- .github/workflows/unit-test-python.yml | 11 ----------- pom.xml | 6 ++---- python/setup.py | 10 +++++----- python/tsfile/__init__.py | 1 - 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/.github/workflows/unit-test-python.yml b/.github/workflows/unit-test-python.yml index f99eb3b4b..1f67f98fa 100644 --- a/.github/workflows/unit-test-python.yml +++ b/.github/workflows/unit-test-python.yml @@ -65,17 +65,6 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} restore-keys: ${{ runner.os }}-m2- - - name: Install dependencies - shell: bash - run: | - if [[ "$RUNNER_OS" == "Linux" ]]; then - if command -v apt-get >/dev/null 2>&1; then - sudo apt-get update - sudo apt-get install -y uuid-dev - elif command -v yum >/dev/null 2>&1; then - sudo yum install -y libuuid-devel - fi - fi # On Windows systems the 'mvnw' script needs an additional ".cmd" appended. - name: Calculate platform suffix id: platform_suffix diff --git a/pom.xml b/pom.xml index 561e95356..fa480b5b0 100644 --- a/pom.xml +++ b/pom.xml @@ -137,8 +137,6 @@ **/third_party/** **/.python-version **/**venv-py**/** - **/.python-version - python/.python-version @@ -757,8 +755,8 @@ win windows-amd64 MinGW Makefiles - - python.exe + venv/Scripts/ + python diff --git a/python/setup.py b/python/setup.py index db95ae6ae..759abab33 100644 --- a/python/setup.py +++ b/python/setup.py @@ -34,12 +34,12 @@ CPP_INC = CPP_OUT / "include" version = "2.2.1.dev" -system = platform.system() -(PKG / "include").mkdir(exist_ok=True) -if (PKG / "include").exists() and CPP_INC.exists(): +if not CPP_INC.exists(): + raise FileNotFoundError(f"missing C++ headers: {CPP_INC}") +if (PKG / "include").exists(): shutil.rmtree(PKG / "include") - shutil.copytree(CPP_INC, PKG / "include") +shutil.copytree(CPP_INC, PKG / "include") if sys.platform.startswith("linux"): candidates = sorted(CPP_LIB.glob("libtsfile.so*"), key=lambda p: len(p.name), reverse=True) if not candidates: @@ -118,7 +118,7 @@ def finalize_options(self): extra_compile_args += ["-O3", "-std=c++11", "-fvisibility=hidden", "-fPIC"] extra_link_args += ["-Wl,-rpath,@loader_path", "-stdlib=libc++"] elif sys.platform == "win32": - libraries = ["Tsfile"] + libraries = ["tsfile"] extra_compile_args += ["-O2", "-std=c++11", "-DSIZEOF_VOID_P=8", "-D__USE_MINGW_ANSI_STDIO=1", "-DMS_WIN64", "-D_WIN64"] extra_link_args += [] diff --git a/python/tsfile/__init__.py b/python/tsfile/__init__.py index 7954a216d..55fa3b9e4 100644 --- a/python/tsfile/__init__.py +++ b/python/tsfile/__init__.py @@ -18,7 +18,6 @@ import ctypes import os -import platform import sys if sys.platform == "win32":