Generating platform-specific wheels

appveyor.yml

@@ -5,86 +5,112 @@ skip_branch_with_pr: true
 
 image: ubuntu
 
-stack: python 3.9
-
 environment:
-  PGLET_VERSION: 0.5.6
-  TWINE_USERNAME: __token__
-  TWINE_NON_INTERACTIVE: true
-  pypi_key: 
-    secure: bPSgMbRTG8EIrSW8RUbb8go0xbmg4/E4NnYfGl8h+L+KlWEk5mHLojLG/b6S6nVSCRXEHU/Vc0KjxLO+6mDrC3OT13kwv3fRRT90JNkOIlxePeoEhRFnh3u9nxlxeWncYBayef3U/zdxTTBd4YXqMPZ22qvCqeIRcOi/fzTRrfZ84Wr5V/JS7Jpmzg2yc2B/kzUBiQlfvflY7YUw/QFaiPhYytIglE7XY+r04JwNfgA=
-  test_pypi_key: 
-    secure: LAsjCb0tqzjUEPxHTq8TPmTDzERQpSS0po96HhXEEczG8EaWqCUdtb+bhT3QLihy+4X4MvErEYXYieUT+KjpuVdgjc/UrqDEuQngBL9r7k5cRMV878c232zbLGljHckNr4TCzHJosANrYBrx0sNg9gVnAoJOxNbu40A1rVB8Qj9ob/eCKwXHHRxfrf0d/NwoLZViCE4toV333tVr5vshlLkQA3oRiS2OZG3VKagTI4nsKCaOFqJHFlvy2a0LEUdc
-
-install:
-- python --version
-- pip install --upgrade setuptools wheel twine pdm
-- pdm install
-- echo Using Pglet ${PGLET_VERSION}
-
-# downloading binaries
-- sh: |
-    function download_binary() {
-      echo "Downloading $1"
-      mkdir -p pglet/bin/$1
-      curl -L https://github.com/pglet/pglet/releases/download/v${PGLET_VERSION}/pglet-${PGLET_VERSION}-$1 -o pglet/bin/$1/pglet
-      chmod a+x pglet/bin/$1/pglet
-    }
-
-    echo "Downloading windows-amd64"
-    mkdir -p pglet/bin/windows-amd64
-    curl -L https://github.com/pglet/pglet/releases/download/v${PGLET_VERSION}/pglet-${PGLET_VERSION}-windows-amd64.exe -o pglet/bin/windows-amd64/pglet.exe
-
-    download_binary "linux-amd64"
-    download_binary "linux-arm64"
-    download_binary "linux-arm"
-    download_binary "darwin-amd64"
-    download_binary "darwin-arm64"
-
-build_script:
-# run tests
-#- pytest
-- ls -alR pglet/bin
-
-# build and publish package
-- ps: |
-    $ErrorActionPreference = "Stop"
-
-    if ($env:APPVEYOR_REPO_TAG -eq 'true') {
-      # release mode
-
-      # version
-      $ver = $env:APPVEYOR_REPO_TAG_NAME
-      if ($ver.StartsWith('v')) { $ver = $ver.Substring(1) }
-
-      # prerelease moniker
-      $idx = $ver.indexOf('-')
-      if ($idx -ne -1) {
-        $prerelease = $ver.Substring($idx + 1)
-        $ver = $ver.Substring(0, $idx)
+  matrix:
+    - job_name: Test Python 3.7
+      job_group: tests
+      python_version: 3.7
+
+    - job_name: Test Python 3.8
+      job_group: tests
+      python_version: 3.8
+
+    - job_name: Test Python 3.9
+      job_group: tests
+      python_version: 3.9
+
+    - job_name: Test Python 3.10
+      job_group: tests
+      python_version: 3.10
+
+    - job_name: Build package
+      job_group: build
+      job_depends_on: tests
+      python_version: 3.10
+      TWINE_USERNAME: __token__
+      TWINE_NON_INTERACTIVE: true
+      pypi_key: 
+        secure: bPSgMbRTG8EIrSW8RUbb8go0xbmg4/E4NnYfGl8h+L+KlWEk5mHLojLG/b6S6nVSCRXEHU/Vc0KjxLO+6mDrC3OT13kwv3fRRT90JNkOIlxePeoEhRFnh3u9nxlxeWncYBayef3U/zdxTTBd4YXqMPZ22qvCqeIRcOi/fzTRrfZ84Wr5V/JS7Jpmzg2yc2B/kzUBiQlfvflY7YUw/QFaiPhYytIglE7XY+r04JwNfgA=
+      test_pypi_key: 
+        secure: LAsjCb0tqzjUEPxHTq8TPmTDzERQpSS0po96HhXEEczG8EaWqCUdtb+bhT3QLihy+4X4MvErEYXYieUT+KjpuVdgjc/UrqDEuQngBL9r7k5cRMV878c232zbLGljHckNr4TCzHJosANrYBrx0sNg9gVnAoJOxNbu40A1rVB8Qj9ob/eCKwXHHRxfrf0d/NwoLZViCE4toV333tVr5vshlLkQA3oRiS2OZG3VKagTI4nsKCaOFqJHFlvy2a0LEUdc
+
+
+stack: python $python_version
+
+for:
+
+###############
+#    Tests    #
+###############
+-
+  matrix:
+    only:
+    - job_group: tests
+
+  install:
+  - python --version
+  - pip install pdm
+  - pdm install
+
+  build: off
+
+  test_script:
+  - pdm run pytest tests
+
+###############
+#    Build    #
+###############
+- 
+  matrix:
+    only:
+    - job_group: build
+
+  install:
+  - python --version
+  - pip install --upgrade setuptools wheel twine pdm
+  - pdm install
+
+  test: off
+
+  build_script:
+  - ps: |
+      $ErrorActionPreference = "Stop"
+
+      if ($env:APPVEYOR_REPO_TAG -eq 'true') {
+        # release mode
+
+        # version
+        $ver = $env:APPVEYOR_REPO_TAG_NAME
+        if ($ver.StartsWith('v')) { $ver = $ver.Substring(1) }
+
+        # prerelease moniker
+        $idx = $ver.indexOf('-')
+        if ($idx -ne -1) {
+          $prerelease = $ver.Substring($idx + 1)
+          $ver = $ver.Substring(0, $idx)
+        }
+        $env:TWINE_PASSWORD = $env:pypi_key
+      } else {
+
+        # build mode
+        $ver = $env:APPVEYOR_BUILD_VERSION
+        $env:TWINE_PASSWORD = $env:test_pypi_key
+        $env:TWINE_REPOSITORY = 'testpypi'
       }
-      $env:TWINE_PASSWORD = $env:pypi_key
-    } else {
-
-      # build mode
-      $ver = $env:APPVEYOR_BUILD_VERSION
-      $env:TWINE_PASSWORD = $env:test_pypi_key
-      $env:TWINE_REPOSITORY = 'testpypi'
-    }
-
-    # patch version
-    (Get-Content pyproject.toml).replace("version = `"0.1.0`"", "version = `"$ver`"") | Set-Content pyproject.toml
 
-# build package
-- pdm build
+      # patch version
+      $env:PACKAGE_VERSION = $ver
+      (Get-Content pyproject.toml).replace("version = `"0.1.0`"", "version = `"$ver`"") | Set-Content pyproject.toml
 
-# publish package
-- sh: |
-    if [[ "$APPVEYOR_PULL_REQUEST_NUMBER" == "" ]]; then
-        twine upload dist/*
-    fi
+  # build package
+  - pdm build
+  - python3 build-wheels.py
 
-test: off
+  # publish package
+  - sh: |
+      if [[ "$APPVEYOR_PULL_REQUEST_NUMBER" == "" ]]; then
+          twine upload dist/*
+      fi
 
-artifacts:
-  path: dist/*
+  artifacts:
+    path: dist/*

build-wheels.py

@@ -0,0 +1,205 @@
+import glob
+import hashlib
+import io
+import os
+import pathlib
+import shutil
+import stat
+import sys
+import urllib.request
+import zipfile
+from base64 import urlsafe_b64encode
+
+packages = {
+    "Windows amd64": {
+        "asset": "windows-amd64.exe",
+        "exec": "pglet.exe",
+        "wheel_tags": ["py3-none-win_amd64"],
+        "file_suffix": "py3-none-win_amd64",
+    },
+    "Linux amd64": {
+        "asset": "linux-amd64",
+        "exec": "pglet",
+        "wheel_tags": [
+            "py3-none-manylinux_2_17_x86_64",
+            "py3-none-manylinux2014_x86_64",
+        ],
+        "file_suffix": "py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64",
+    },
+    "Linux arm64": {
+        "asset": "linux-arm64",
+        "exec": "pglet",
+        "wheel_tags": [
+            "py3-none-manylinux_2_17_aarch64",
+            "py3-none-manylinux2014_aarch64",
+        ],
+        "file_suffix": "py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64",
+    },
+    "Linux arm": {
+        "asset": "linux-arm",
+        "exec": "pglet",
+        "wheel_tags": [
+            "py3-none-manylinux_2_17_armv7l",
+            "py3-none-manylinux2014_armv7l",
+        ],
+        "file_suffix": "py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l",
+    },
+    "macOS amd64": {
+        "asset": "darwin-amd64",
+        "exec": "pglet",
+        "wheel_tags": ["py3-none-macosx_10_14_x86_64"],
+        "file_suffix": "py3-none-macosx_10_14_x86_64",
+    },
+    "macOS arm64": {
+        "asset": "darwin-arm64",
+        "exec": "pglet",
+        "wheel_tags": ["py3-none-macosx_12_0_arm64"],
+        "file_suffix": "py3-none-macosx_12_0_arm64",
+    },
+}
+
+
+def unpack_zip(zip_path, dest_dir):
+    zf = zipfile.ZipFile(zip_path)
+    zf.extractall(path=dest_dir)
+
+
+def download_pglet(version, suffix, dest_file):
+    file_name = f"pglet-{version}-{suffix}"
+    pglet_url = (
+        f"https://github.com/pglet/pglet/releases/download/v{version}/{file_name}"
+    )
+    print(f"Downloading {pglet_url}...")
+    urllib.request.urlretrieve(pglet_url, dest_file)
+    st = os.stat(dest_file)
+    os.chmod(dest_file, st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+
+
+def get_pglet_version(current_dir):
+    constants_py = current_dir.joinpath("pglet", "constants.py")
+
+    version_prefix = "PGLET_SERVER_VERSION"
+    with open(constants_py) as yml:
+        for line in yml:
+            if version_prefix in line:
+                return line.split("=")[1].strip().strip('"')
+    raise f"{version_prefix} not found in constants.py"
+
+
+def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE):
+    """Yield pieces of data from a file-like object until EOF."""
+    while True:
+        chunk = file.read(size)
+        if not chunk:
+            break
+        yield chunk
+
+
+def rehash(path, blocksize=1 << 20):
+    """Return (hash, length) for path using hashlib.sha256()"""
+    h = hashlib.sha256()
+    length = 0
+    with open(path, "rb") as f:
+        for block in read_chunks(f, size=blocksize):
+            length += len(block)
+            h.update(block)
+    digest = "sha256=" + urlsafe_b64encode(h.digest()).decode("latin1").rstrip("=")
+    # unicode/str python2 issues
+    return (digest, str(length))  # type: ignore
+
+
+current_dir = pathlib.Path(os.getcwd())
+print("current_dir", current_dir)
+
+whl_files = glob.glob(str(current_dir.joinpath("dist", "*.whl")))
+if len(whl_files) == 0:
+    print("No .whl files found. Run 'pdm build' first.")
+    sys.exit(1)
+
+orig_whl = whl_files[0]
+
+package_version = os.path.basename(orig_whl).split("-")[1]
+pglet_version = get_pglet_version(current_dir)
+
+print("package_version", package_version)
+print("pglet_version", pglet_version)
+
+for name, package in packages.items():
+    print(f"Building {name}...")
+
+    print("Unpacking original wheel file...")
+    unpacked_whl = current_dir.joinpath("dist", "wheel")
+    unpacked_whl.mkdir(exist_ok=True)
+    unpack_zip(orig_whl, unpacked_whl)
+
+    # read original WHEEL file omitting tags
+    wheel_path = str(
+        current_dir.joinpath(
+            "dist", "wheel", f"pglet-{package_version}.dist-info", "WHEEL"
+        )
+    )
+    wheel_lines = []
+
+    with open(wheel_path, "r") as f:
+        for line in f.readlines():
+            if not "Tag: " in line:
+                wheel_lines.append(line)
+
+    # print(wheel_lines)
+
+    # read original RECORD file
+    record_path = str(
+        current_dir.joinpath(
+            "dist", "wheel", f"pglet-{package_version}.dist-info", "RECORD"
+        )
+    )
+    record_lines = []
+
+    with open(record_path, "r") as f:
+        for line in f.readlines():
+            if not "dist-info/WHEEL," in line:
+                record_lines.append(line)
+
+    # print(record_lines)
+
+    # create "bin" directory
+    bin_path = current_dir.joinpath("dist", "wheel", "pglet", "bin")
+    bin_path.mkdir(exist_ok=True)
+    asset = package["asset"]
+    exec_filename = package["exec"]
+    exec_path = str(bin_path.joinpath(exec_filename))
+    download_pglet(pglet_version, asset, exec_path)
+
+    # update RECORD
+    h, l = rehash(exec_path)
+    record_lines.insert(len(record_lines) - 3, f"pglet/bin/{exec_filename},{h},{l}\n")
+    # for line in record_lines:
+    #     print(line.strip())
+
+    # update WHEEL file
+    for tag in package["wheel_tags"]:
+        wheel_lines.append(f"Tag: {tag}\n")
+
+    # save WHEEL
+    with open(wheel_path, "w") as f:
+        f.writelines(wheel_lines)
+
+    # update RECORD
+    h, l = rehash(wheel_path)
+    record_lines.insert(
+        len(record_lines) - 3,
+        f"pglet-{package_version}.dist-info/WHEEL,{h},{l}\n",
+    )
+
+    # save RECORD
+    with open(record_path, "w") as f:
+        f.writelines(record_lines)
+
+    # zip
+    suffix = package["file_suffix"]
+    zip_filename = current_dir.joinpath("dist", f"pglet-{package_version}-{suffix}")
+    shutil.make_archive(zip_filename, "zip", unpacked_whl)
+    os.rename(f"{zip_filename}.zip", f"{zip_filename}.whl")
+
+    # cleanup
+    shutil.rmtree(str(unpacked_whl))