From 8b592333e5dda5eab20a07e21681a5a97b35c9ca Mon Sep 17 00:00:00 2001 From: Ryan Mast Date: Mon, 3 Feb 2025 22:01:46 -0800 Subject: [PATCH] Update clang wheel packaging and add CI workflows --- .github/dependabot.yml | 11 ++ .github/workflows/llvm-update-autocheck.yml | 46 ++++++++ .github/workflows/release.yml | 47 ++++++++ CMakeLists.txt | 28 +++++ README.md | 22 ++-- llvm_version.cmake | 2 + make-from-packages.py | 118 -------------------- packages.lst | 14 --- pyproject.toml | 41 +++++++ requirements.txt | 3 - setup.py.tpl | 35 ------ 11 files changed, 190 insertions(+), 177 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/llvm-update-autocheck.yml create mode 100644 .github/workflows/release.yml create mode 100644 CMakeLists.txt create mode 100644 llvm_version.cmake delete mode 100644 make-from-packages.py delete mode 100644 packages.lst create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py.tpl diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d5bbc30 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/llvm-update-autocheck.yml b/.github/workflows/llvm-update-autocheck.yml new file mode 100644 index 0000000..fb6115c --- /dev/null +++ b/.github/workflows/llvm-update-autocheck.yml @@ -0,0 +1,46 @@ +name: Auto-update LLVM Version +on: + workflow_dispatch: + schedule: + - cron: '0 10 * * 1' +jobs: + update-dep: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Check and update LLVM version + id: check-llvm + run: | + old_version="$(grep -Po '(?<=LLVM_VERSION )\d+(\.\d+)+' llvm_version.cmake)" + echo "Old version: $old_version" + echo "old_version=$old_version" >> $GITHUB_OUTPUT + + new_version=$(git -c 'versionsort.suffix=-' \ + ls-remote --exit-code --refs --sort='version:refname' --tags https://github.com/llvm/llvm-project 'llvmorg-*.*.*'\ + | grep -Po '\d+\.\d+(\.\d+)?$' \ + | tail --lines=1) + + echo "New version: $new_version" + echo "new_version=$new_version" >> $GITHUB_OUTPUT + + if [ "$old_version" != "$new_version" ]; then + echo "set(LLVM_VERSION $new_version)" > llvm_version.cmake + curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-${new_version}/clang-${new_version}.src.tar.xz -o clang-${new_version}.src.tar.xz + hash=$(sha256sum clang-${new_version}.src.tar.xz | cut -d ' ' -f1) + echo "set(LLVM_SHA256 $hash)" >> llvm_version.cmake + echo "LLVM version updated" + else + echo "No change in LLVM version" + fi + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Update LLVM to version ${{ steps.check-llvm.outputs.new_version }} + title: Update LLVM version (${{ steps.check-llvm.outputs.new_version }}) + body: | + - Update LLVM version to ${{ steps.check-llvm.outputs.new_version }} + Auto-generated by ${{ github.server_url }}/${{ github.repository }}/runs/${{ github.job }}?check_suite_focus=true + author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + branch: llvm-version-update/${{ steps.check-llvm.outputs.new_version }} \ No newline at end of file diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..bc9510e --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,47 @@ +name: Build and Release on PyPI + +on: + workflow_dispatch: + push: + tags: + - v* + +jobs: + build-release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Build wheel + run: | + pip install build twine + python -m build + python -m twine check dist/* + - name: Upload Python package dist artifacts + uses: actions/upload-artifact@v4 + with: + name: python-package-dist + path: dist + + pypi-publish: + name: Upload release to PyPI + runs-on: ubuntu-latest + needs: build-release + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') + environment: + name: pypi + url: https://pypi.org/p/clang + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + - name: Download Python package dist artifacts + uses: actions/download-artifact@v4 + with: + name: python-package-dist + path: dist + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..8d8a1f7 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.15) +project(clang NONE) + +include(ExternalProject) + +include(llvm_version.cmake) + +# Set the URL and local paths +set(CLANG_TARBALL_URL "https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/clang-${LLVM_VERSION}.src.tar.xz") +set(CLANG_TARBALL "${CMAKE_BINARY_DIR}/clang-${LLVM_VERSION}.src.tar.xz") +set(CLANG_SOURCE_DIR "${CMAKE_BINARY_DIR}/clang-src") +set(PYTHON_BINDINGS_OUTPUT "${CLANG_SOURCE_DIR}/bindings/python/clang/") + +# Download the tarball using ExternalProject_Add +ExternalProject_Add(clang + URL ${CLANG_TARBALL_URL} + URL_HASH SHA256=${LLVM_SHA256} + DOWNLOAD_DIR ${CMAKE_BINARY_DIR} + SOURCE_DIR ${CLANG_SOURCE_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + LOG_DOWNLOAD OFF + DOWNLOAD_EXTRACT_TIMESTAMP ON +) + +# Define installation so that scikit-build packages the python files into the wheel +install(DIRECTORY ${PYTHON_BINDINGS_OUTPUT} DESTINATION clang USE_SOURCE_PERMISSIONS) \ No newline at end of file diff --git a/README.md b/README.md index dbd9ac7..50c44f2 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,24 @@ -Clang Python ppackage for pypi ------------------------------- +# Clang Python package for PyPI This is the python bindings subdir of llvm clang repository. https://github.com/llvm/llvm-project/tree/main/clang/bindings/python The debian packages are pulled from llvm repo, extracted and pushed to pypi. -Installation ------------- +## Installation -`pip install clang` +You can install the package using pip: -For a specific version +```bash +pip install clang +``` -`pip install clang==14` \ No newline at end of file +Or for a specific version: + +```bash +pip install clang==14 +``` + +## License + +This project is licensed under the Apache-2.0 License with LLVM exception. See the LICENSE file for more details. \ No newline at end of file diff --git a/llvm_version.cmake b/llvm_version.cmake new file mode 100644 index 0000000..4e09681 --- /dev/null +++ b/llvm_version.cmake @@ -0,0 +1,2 @@ +set(LLVM_VERSION 18.1.8) +set(LLVM_SHA256 5724fe0a13087d5579104cedd2f8b3bc10a212fb79a0fcdac98f4880e19f4519) \ No newline at end of file diff --git a/make-from-packages.py b/make-from-packages.py deleted file mode 100644 index e3f25cb..0000000 --- a/make-from-packages.py +++ /dev/null @@ -1,118 +0,0 @@ -import csv -import logging -import os -import shutil -import subprocess - -import requests - - -class PackageBuilder: - code_folder = 'clang' - temp_folder = 'tmp' - temp_package_filename = 'tmp_llvm_clang.deb' - python_version = 'python3' - setup_file = 'setup.py' - setup_file_tpl = 'setup.py.tpl' - - def __init__(self, version, pyversion, url): - self.version = version.strip() - self.pyversion = str(int(pyversion.strip())) - self.url = url.strip() - - def do(self): - logging.info(f'packaging version {self.version} from {self.url}') - if self.pyversion == '2': - self.python_version = 'python2.7' - else: - self.python_version = 'python3' - self.clean() - self.download() - self.unpack() - try: - self.move() - except FileNotFoundError as e: - logging.error(f"error in package v:{self.version} pyv:{self.pyversion} url:{self.url}") - raise RuntimeError( - f"Package python-clang version {self.version} for python {self.pyversion} does " - f"not contain clang python binding.") from e - self.setup() - self.build() - self.upload() - logging.info(f'packaged version {self.version} for python {self.pyversion}') - - def clean(self): - for d in [self.code_folder, self.temp_folder]: - if os.path.exists(d): - shutil.rmtree(d) - if os.path.exists(self.setup_file): - os.remove(self.setup_file) - logging.debug("Working folders removed") - - def download(self): - response = requests.get(self.url) - logging.debug(f"download ret {response.status_code}") - if response.status_code != 200: - raise RuntimeError("download failed") - if not os.path.exists(self.temp_folder): - os.makedirs(self.temp_folder) - filename = os.path.join(self.temp_folder, self.temp_package_filename) - with open(filename, 'wb') as out: - out.write(response.content) - return filename - - def unpack(self): - ret = subprocess.run(['dpkg-deb', '-xv', self.temp_package_filename, '.'], cwd=self.temp_folder, - capture_output=True) - logging.debug(f"unpack ret {ret}") - if ret.returncode != 0: - logging.error(ret.stderr) - raise RuntimeError("unpack failed: ") - - def move(self): - src = os.path.join(self.temp_folder, 'usr', 'lib', self.python_version, 'dist-packages', 'clang') - dst = '.' - shutil.move(src, dst) - logging.debug(f"python package moved {src} -> {dst}") - - def setup(self): - with open(self.setup_file_tpl, 'r') as tpl: - tpl_content = tpl.read() - tpl_content = tpl_content.replace('%VERSION%', self.version) - tpl_content = tpl_content.replace('%PYTHON_VERSION%', self.pyversion) - with open(self.setup_file, 'w') as setup_file: - setup_file.write(tpl_content) - logging.debug(f"setup.py file writen for clang v:{self.version}, pyv:{self.pyversion}") - - def build(self): - # https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html - ret = subprocess.run(['python', '-m', 'build'], capture_output=True) - logging.debug(f"build ret {ret}") - if ret.returncode != 0: - logging.error(ret.stderr) - raise RuntimeError("build failed") - - def upload(self): - # https://blog.ganssle.io/articles/2021/10/setup-py-deprecated.html - ret = subprocess.run(['twine', 'upload', f'dist/clang-{self.version}*'], capture_output=True) - logging.debug(f"upload ret {ret}") - if ret.returncode != 0: - logging.error(ret.stderr) - raise RuntimeError("upload failed") - - -def main(): - logging.basicConfig(level=logging.INFO) - - with open('packages.lst', newline='') as csvfile: - reader = csv.reader(csvfile) - headers = next(reader) - for version, pyversion, url in reader: - if version.startswith('#'): - continue - builder = PackageBuilder(version, pyversion, url) - builder.do() - - -if __name__ == '__main__': - main() diff --git a/packages.lst b/packages.lst deleted file mode 100644 index edf86da..0000000 --- a/packages.lst +++ /dev/null @@ -1,14 +0,0 @@ -version, pyversion, url -#17.0.6, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-17/python3-clang-17_17.0.6~%2B%2B20231205064722%2B6009708b4367-1~exp1~20231205064821.75_amd64.deb -#16.0.6, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-16/python3-clang-16_16.0.6~%2b%2b20230908115458%2b7cbf1a259152-1~exp1~20230908115553.113_amd64.deb -#16.0.1, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-16/python3-clang-16_16.0.1~%2b%2b20230328073207%2b42d1b276f779-1~exp1~20230328073220.62_amd64.deb -#15.0.7, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-15/python3-clang-15_15.0.7~%2b%2b20230317110556%2b8dfdcc7b7bf6-1~exp1~20230317110643.127_amd64.deb -#14.0.6, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-14/python3-clang-14_14.0.6~%2b%2b20230130095011%2bf28c006a5895-1~exp1~20230130215051.177_amd64.deb -#13.0.1, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-13/python3-clang-13_13.0.1~%2b%2b20220126091935%2b75e33f71c2da-1~exp1~20220126212020.65_amd64.deb -#12.0.1, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-12/python3-clang-12_12.0.1~%2b%2b20211029101259%2bfed41342a82f-1~exp1~20211029221820.5_amd64.deb -#11.1, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-11/python3-clang-11_11.1.0~%2b%2b20211114101220%2b1fdec59bffc1-1~exp1~20211114221643.10_amd64.deb -#11.0, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-11/python3-clang-11_11.0.0~%2b%2b20200808123325%2b279922f108c-1~exp1~20200807223952.39_i386.deb -#10.0.1, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-10/python3-clang-10_10.0.1~%2b%2b20200809072601%2bef32c611aa2-1~exp1~20200809173202.199_i386.deb -#9.0, 2, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-9/python-clang-9_9~%2b20191211112125%2bc1a0a213378-1~exp1~20191211223511.119_i386.deb -#8.0.1, 3, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-8/python3-clang-8_8.0.1%2bsvn369350-1~exp1~20200303215641.87_i386.deb -#7.1.0, 2, https://apt.llvm.org/unstable/pool/main/l/llvm-toolchain-7/python-clang-7_7.1.0~svn353565-1~exp1~20190406085125.68_i386.deb diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..66814cf --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["scikit-build-core"] +build-backend = "scikit_build_core.build" + +[project] +name = "clang" +description = "libclang python bindings" +readme = "README.md" +license = { text = "Apache-2.0 with LLVM exception" } +authors = [ + { name = "LLVM team - pypi upload by Loic Jaquemet" } +] +keywords = ["llvm", "clang", "libclang"] +classifiers = [ + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Development Status :: 5 - Production/Stable", + "Topic :: Software Development :: Compilers", + "Programming Language :: Python :: 3", +] +dynamic = ["version"] + +[project.urls] +Homepage = "http://clang.llvm.org/" +Download = "http://llvm.org/releases/download.html" + +[dependecy-groups] +dev = ["build", "twine"] + +[tool.scikit-build] +wheel.py-api = "py3" +wheel.platlib = false +wheel.license-files = [] + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "llvm_version.cmake" +regex = '''(?sx) +set\(\s*LLVM_VERSION\s+(?P\d+(?:\.\d+)+)\s*\) +''' +result = "{version}" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index a82266c..0000000 --- a/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -requests -build -twine \ No newline at end of file diff --git a/setup.py.tpl b/setup.py.tpl deleted file mode 100644 index dc56421..0000000 --- a/setup.py.tpl +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -from setuptools import setup - -long_description="""This is the python bindings subdir of llvm clang repository. -https://github.com/llvm/llvm-project/tree/main/clang/bindings/python - -This is a non-official packaging directly from the debian packages for the purpose of pypi package. -""" - -setup( - name="clang", - version="%VERSION%", - description="libclang python bindings", - long_description=long_description, - url="http://clang.llvm.org/", - download_url="http://llvm.org/releases/download.html", - license="Apache-2.0 with LLVM exception", - classifiers=[ - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Development Status :: 5 - Production/Stable", - "Topic :: Software Development :: Compilers", - "Programming Language :: Python :: %PYTHON_VERSION%", - ], - keywords=["llvm", "clang", "libclang"], - author="LLVM team - pypi upload by Loic Jaquemet", - zip_safe=False, - packages=["clang"], - # if use nose.collector, many plugins not is avaliable - # see: http://nose.readthedocs.org/en/latest/setuptools_integration.html - #test_suite="nose.collector", - #tests_require=['nose'] -)