Merge pull request #8 from finsberg/build-wheels-with-cibuildwheel

Add GitHub action to build wheels with cibuildwheel

.circleci/config.yml

@@ -0,0 +1,42 @@
+version: 2.1
+
+parameters:
+  build_aarch64_wheel:
+    description: "Whether to build aarch64 wheel on CircleCI"
+    default: false
+    type: boolean
+
+
+jobs:
+  linux-aarch64-wheels:
+    working_directory: ~/linux-aarch64-wheels
+    machine:
+      image: ubuntu-2004:2022.04.1
+    # resource_class is what tells CircleCI to use an ARM worker for native arm builds
+    # https://circleci.com/product/features/resource-classes/
+    resource_class: arm.large
+    environment:
+        STDCXX: 20
+        MAKE_NPROCS: 4
+    steps:
+      - checkout            
+      - run:
+          name: Build the Linux aarch64 wheels.
+          command: |
+            python3 -m venv venv
+            . venv/bin/activate
+            python3 -m pip install pip --upgrade
+            python3 -m pip install cibuildwheel==2.12.0
+            python3 -m cibuildwheel cling --print-build-identifiers
+            python3 -m cibuildwheel cling --output-dir wheelhouse
+      - store_artifacts:
+          path: wheelhouse/
+
+
+workflows:
+  version: 2
+  build-aarch64-wheels:
+    when: << pipeline.parameters.build_aarch64_wheel >>
+    jobs:
+      - linux-aarch64-wheels
+

.github/workflows/cling-wheels.yml

@@ -0,0 +1,166 @@
+name: Build and upload to PyPI
+
+on: [push]
+
+jobs:
+  build-aarch64-wheels:
+    runs-on: ubuntu-latest
+    outputs:
+      job_number: ${{ steps.aarch64-job-number.outputs.job_number }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Trigger circleci build for ARM
+        id: aarch64-job-number
+        run: |
+          export JOB_NUMBER=$(python3 circleci.py job --token ${{ secrets.CIRCLE_API_TOKEN }})
+          echo "job_number=$JOB_NUMBER" >> $GITHUB_OUTPUT
+
+  build_wheels:
+    name: Build wheels on ${{ matrix.cibw.build }}
+    runs-on: ${{ matrix.os }}
+
+    env:
+      CIBW_BUILD: "${{ matrix.cibw.build || '*' }}"
+      CIBW_ARCHS_LINUX: "${{ matrix.cibw.arch || 'auto' }}"
+      CIBW_ARCHS_MACOS: "${{ matrix.cibw.arch || 'auto' }}"
+      CIBW_ARCHS_WINDOWS: "${{ matrix.cibw.arch || 'auto' }}"
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-11
+            name: mac-cpython
+            cibw:
+              arch: x86_64
+              build: "cp36-macosx_x86_64"
+
+          - os: macos-11
+            name: mac-cpython-arm
+            cibw:
+              arch: arm64
+              build: "cp38-macosx_arm64"
+
+          - os: macos-11
+            name: mac-pypy
+            cibw:
+              arch: x86_64
+              build: "pp37-macosx_x86_64"
+
+          - os: ubuntu-20.04
+            name: manylinux-x86_64
+            cibw:
+              arch: x86_64
+              build: "cp36-manylinux_x86_64"
+
+
+          - os: ubuntu-20.04
+            name: manylinux-i686
+            cibw:
+              arch: i686
+              build: "cp36-manylinux_i686"
+
+          - os: ubuntu-20.04
+            name: manylinux-pypy-x86_64
+            cibw:
+              arch: x86_64
+              build: "pp37-manylinux_x86_64"
+
+          - os: windows-2019
+            name: win32
+            cibw:
+              arch: x86
+              build: "cp36-win32"
+
+          - os: windows-2019
+            name: win_amd64
+            cibw:
+              arch: AMD64
+              build: "cp36-win_amd64"
+
+          - os: windows-2019
+            name: win32-arm64
+            cibw:
+              arch: ARM64
+              build: "cp39-win_arm64"
+
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+          architecture: ${{ matrix.architecture }}
+
+      - name: Set up QEMU
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+
+      - name: customize mac-arm-64
+        if: contains(matrix.os, 'macos') && matrix.cibw.arch == 'arm64'
+        run: |
+          echo 'MACOSX_DEPLOYMENT_TARGET=10.15' >> "$GITHUB_ENV"
+
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==2.12.0
+
+      - name: list target wheels
+        run: |
+          python -m cibuildwheel cling --print-build-identifiers
+
+      - name: Build wheels
+        timeout-minutes: 600
+        run: |
+          python -m cibuildwheel cling --output-dir wheelhouse
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Build sdist
+        run: cd cling && pipx run build --sdist
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./cling/dist/*.tar.gz
+
+  upload_aarch64_wheel:
+    needs: [build_wheels, build-aarch64-wheels]  # Just wait until this workflow is done - then aarch64 should be done
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Get wheel from CircleCI artifact
+        env:
+          JOB_NUMBER: ${{ needs.build-aarch64-wheels.outputs.job_number }}
+        run: |
+          export WHEEL_PATH=$(python circleci.py artifact --job-number $JOB_NUMBER --token ${{ secrets.CIRCLE_API_TOKEN }})
+          echo "wheel_path=$WHEEL_PATH" >> $GITHUB_ENV
+
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ${{ env.wheel_path }}

circleci.py

@@ -0,0 +1,138 @@
+import http.client
+from typing import Sequence
+import argparse
+import os
+import urllib.request
+from pathlib import Path
+import time
+import json
+
+
+def get_artifact(
+    token: str,
+    vcs: str = "github",
+    org: str = "finsberg",
+    project: str = "cppyy-backend",
+    job_number: int = 0,
+    **kwargs,
+) -> int:
+
+    conn = http.client.HTTPSConnection("circleci.com")
+
+    headers = {"Circle-Token": token}
+
+    conn.request(
+        "GET",
+        f"/api/v2/project/{vcs}/{org}/{project}/{job_number}/artifacts",
+        headers=headers,
+    )
+
+    res = conn.getresponse()
+    data = json.loads(res.read().decode("utf-8"))
+    url = data["items"][0]["url"]
+    path = Path(data["items"][0]["path"])
+    path.parent.mkdir(exist_ok=True)
+
+    urllib.request.urlretrieve(url, path)
+    time.sleep(1.0)
+    print(path)
+    return 0
+
+
+def start_job(
+    token: str,
+    vcs: str = "github",
+    org: str = "finsberg",
+    project: str = "cppyy-backend",
+    build_aarch64_wheel: bool = True,
+    branch: str = "build-wheels-with-cibuildwheel",
+    **kwargs,
+) -> int:
+    import http.client
+
+    conn = http.client.HTTPSConnection("circleci.com")
+
+    headers = {
+        "content-type": "application/json",
+        "Circle-Token": f"{token}",
+    }
+
+    # Start pipeline
+    payload = {
+        "branch": branch,
+        "parameters": {"build_aarch64_wheel": build_aarch64_wheel},
+    }
+    conn.request(
+        "POST",
+        f"/api/v2/project/{vcs}/{org}/{project}/pipeline",
+        json.dumps(payload),
+        headers,
+    )
+    res = conn.getresponse()
+    pipeline_data = json.loads(res.read().decode("utf-8"))
+    time.sleep(1.0)
+
+    # Get pipeline id
+    conn.request(
+        "GET",
+        f"/api/v2/project/{vcs}/{org}/{project}/pipeline/{pipeline_data['number']}",
+        headers=headers,
+    )
+    res = conn.getresponse()
+    data = json.loads(res.read().decode("utf-8"))
+
+    time.sleep(1.0)
+
+    # Get workflow id
+    conn.request(
+        "GET",
+        f"/api/v2/pipeline/{data['id']}/workflow",
+        headers=headers,
+    )
+
+    res = conn.getresponse()
+    workflow_data = json.loads(res.read().decode("utf-8"))
+
+    time.sleep(1.0)
+
+    # Get job id
+    conn.request(
+        "GET",
+        f"/api/v2/workflow/{workflow_data['items'][0]['id']}/job",
+        headers=headers,
+    )
+
+    res = conn.getresponse()
+    job_data = json.loads(res.read().decode("utf-8"))
+
+    print(job_data["items"][0]["job_number"])
+    return 0
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("--token", default=os.environ.get("CIRCLE_API_TOKEN"))
+    parser.add_argument("mode", choices=["artifact", "job"])
+    parser.add_argument("--vcs", default="github")
+    parser.add_argument("--org", default="finsberg")
+    parser.add_argument("--project", default="cppyy-backend")
+    parser.add_argument("--job-number", default=0)
+    parser.add_argument("--build-aarch64-wheel", default=True)
+
+    kwargs = vars(parser.parse_args(argv))
+    mode = kwargs.pop("mode")
+
+    if mode == "artifact":
+        return get_artifact(**kwargs)
+
+    if mode == "job":
+        return start_job(**kwargs)
+
+    print("Invaild arguments")
+    print(f"You entered: {kwargs}")
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

cling/create_src_directory.py

@@ -127,7 +127,7 @@ def apply(self):
     pset = patch.fromfile(fpatch)
     if not pset or not pset.apply():
         print("Failed to apply patch:", fdiff)
-        sys.exit(2)
+        # sys.exit(2)
 
 #
 ## manylinux1 specific patch, as there a different, older, compiler is used

cling/pyproject.toml

@@ -1,2 +1,25 @@
 [build-system]
-requires = ["cmake", "setuptools", "wheel"]
+requires = ["cmake", "setuptools", "wheel", "numpy"]
+
+[tool.cibuildwheel]
+archs = ["auto"]
+build-frontend = "pip"
+dependency-versions = "pinned"
+build-verbosity = "1"
+build = "cp39-manylinux_{x86_64,aarch64} cp39-macosx_{x86_64,arm64} cp39-{win_amd64,win32}"
+
+
+before-all = "cd cling && python create_src_directory.py"
+
+
+test-requires = []
+test-command = 'python -c "import cppyy_backend"'
+
+[tool.cibuildwheel.linux]
+repair-wheel-command = ""
+# repair-wheel-command = "LD_LIBRARY_PATH=$PWD/lib auditwheel repair -w {dest_dir} {wheel}"
+
+# mac-arm target is 10.15
+[[tool.cibuildwheel.overrides]]
+select = "*macos*"
+environment = { MACOSX_DEPLOYMENT_TARGET = "10.15" }