From 39cbd08289ecbbbd3e8c326aa3046d582aac7a1f Mon Sep 17 00:00:00 2001 From: Jacob Roberts Date: Thu, 3 Nov 2022 14:39:46 -0700 Subject: [PATCH 1/7] Add CI for testing the built wheels --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 420d116..bdd3555 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,6 @@ jobs: pip install -r requirements.txt - name: Build Wheels run: | - mkdir dist python make_wheels.py - name: Show built files run: | From 3eb29da6da1bc7f16018eac2b9c1185034b06694 Mon Sep 17 00:00:00 2001 From: Jacob Roberts Date: Thu, 3 Nov 2022 13:48:51 -0700 Subject: [PATCH 2/7] Add initial support for downloading musl-built nodejs from the unofficial builds --- .github/workflows/test.yaml | 1 + .gitignore | 6 +++++- make_wheels.py | 33 +++++++++++++++++++++++++++------ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bdd3555..420d116 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,6 +23,7 @@ jobs: pip install -r requirements.txt - name: Build Wheels run: | + mkdir dist python make_wheels.py - name: Show built files run: | diff --git a/.gitignore b/.gitignore index d23485d..a46a752 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,8 @@ nodejs-cmd/*.egg-info .DS_Store env*/ __pycache__/ -*.py[cod] \ No newline at end of file +*.py[cod] +venv +package.json +node_modules +package-lock.json diff --git a/make_wheels.py b/make_wheels.py index ed21e0d..c48dec7 100644 --- a/make_wheels.py +++ b/make_wheels.py @@ -1,5 +1,6 @@ import os import hashlib +import pathlib import urllib.request import libarchive from email.message import EmailMessage @@ -45,8 +46,17 @@ 'linux-x64': 'manylinux_2_12_x86_64.manylinux2010_x86_64', 'linux-armv7l': 'manylinux_2_17_armv7l.manylinux2014_armv7l', 'linux-arm64': 'manylinux_2_17_aarch64.manylinux2014_aarch64', + 'linux-x64-musl': 'musllinux_1_1_x86_64' } +# https://github.com/nodejs/unofficial-builds/ +# Versions added here should match the keys above +UNOFFICIAL_NODEJS_BUILDS = {'linux-x64-musl'} + +_mismatched_versions = UNOFFICIAL_NODEJS_BUILDS - set(PLATFORMS.keys()) +if _mismatched_versions: + raise Exception(f"A version mismatch occurred. Check the usage of {_mismatched_versions}") + class ReproducibleWheelFile(WheelFile): def writestr(self, zinfo, *args, **kwargs): @@ -113,6 +123,11 @@ def write_nodejs_wheel(out_dir, *, node_version, version, platform, archive): entry_points = {} init_imports = [] + # Create the output directory if it does not exist + out_dir_path = pathlib.Path(out_dir) + if not out_dir_path.exists(): + out_dir_path.mkdir(parents=True) + with libarchive.memory_reader(archive) as archive: for entry in archive: entry_name = '/'.join(entry.name.split('/')[1:]) @@ -246,13 +261,16 @@ def main() -> None: """).encode('ascii') contents['nodejs/__init__.py'] = (cleandoc(""" + import sys from .node import path as path, main as main, call as call, run as run, Popen as Popen - {init_imports} + if not '-m' in sys.argv: + {init_imports} __version__ = "{version}" node_version = "{node_version}" """)).format( - init_imports='\n'.join(init_imports), + # Note: two space indentation above and below is necessary to align + init_imports='\n '.join(init_imports), version=version, node_version=node_version, ).encode('ascii') @@ -294,10 +312,13 @@ def make_nodejs_version(node_version, suffix=''): print('Suffix:', suffix) for node_platform, python_platform in PLATFORMS.items(): - print(f'- Making Wheel for {node_platform}') - node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.' + \ - ('zip' if node_platform.startswith('win-') else 'tar.xz') - + filetype = 'zip' if node_platform.startswith('win-') else 'tar.xz' + if node_platform in UNOFFICIAL_NODEJS_BUILDS: + node_url = f'https://unofficial-builds.nodejs.org/download/release/v{node_version}/node-v{node_version}-{node_platform}.{filetype}' + else: + node_url = f'https://nodejs.org/dist/v{node_version}/node-v{node_version}-{node_platform}.{filetype}' + + print(f'- Making Wheel for {node_platform} from {node_url}') try: with urllib.request.urlopen(node_url) as request: node_archive = request.read() From 0204b18a711516a60473815175a7c4d8e9ae117b Mon Sep 17 00:00:00 2001 From: Jacob Roberts Date: Thu, 3 Nov 2022 14:55:24 -0700 Subject: [PATCH 3/7] Throw an error if any warnings are emitted when invoking our own package --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 420d116..2e319d7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -77,7 +77,7 @@ jobs: pip install dist\nodejs_bin-${{matrix.nodejs-version}}a3-py3-none-win_amd64.whl - name: Test Package run: - python -m nodejs --version - python -m nodejs.npm --version + python -W error -m nodejs --version + python -W error -m nodejs.npm --version From 8b868793caa86caca86905b713cbe28ef9cbf346 Mon Sep 17 00:00:00 2001 From: Jacob Roberts Date: Thu, 3 Nov 2022 15:36:34 -0700 Subject: [PATCH 4/7] Add testing for variations of operating systems in docker --- .dockerignore | 2 ++ .github/workflows/test.yaml | 41 +++++++++++++++++++++++++++++++++++-- Dockerfile | 26 +++++++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0d0d4f0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +* +!dist diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2e319d7..7de1123 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -43,7 +43,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - nodejs-version: ['16.15.1', '14.19.3', '18.4.0'] + nodejs-version: ['14.19.3', '16.15.1', '18.4.0'] python-version: ['3.7', '3.8', '3.9', '3.10'] steps: @@ -79,5 +79,42 @@ jobs: run: python -W error -m nodejs --version python -W error -m nodejs.npm --version + test-docker: + name: "Test Docker OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}" + runs-on: ubuntu-latest + needs: [build-wheels] + strategy: + fail-fast: false + matrix: + os-variant: [alpine, slim-buster, slim-bullseye] + python-version: ['3.7', '3.8', '3.9', '3.10'] + nodejs-version: ['14.19.3', '16.15.1', '18.4.0'] - + steps: + - uses: actions/checkout@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + with: + platforms: arm64 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + with: + install: true + - uses: actions/download-artifact@v3 + with: + name: nodejs-pip-wheels + path: dist + - name: Docker build + run: | + if [[ ${{ matrix.os-variant }} =~ "alpine" ]]; then + WHEEL_TO_INSTALL=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none-musllinux_1_1_x86_64.whl + else + WHEEL_TO_INSTALL=nodejs_bin-${{ matrix.nodejs-version }}a3-py3-none-manylinux_2_12_x86_64.manylinux2010_x86_64.whl + fi + echo "WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL}" + docker build \ + -f Dockerfile \ + --build-arg PYTHON_VERSION=${{ matrix.python-version }} \ + --build-arg OS_VARIANT=${{ matrix.os-variant }} \ + --build-arg WHEEL_TO_INSTALL=${WHEEL_TO_INSTALL} \ + . diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..97f7286 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +ARG PYTHON_VERSION=3.10 +ARG OS_VARIANT=bullseye-slim + +FROM python:${PYTHON_VERSION}-${OS_VARIANT} + +ARG PYTHON_VERSION +ENV PYTHON_VERSION=${PYTHON_VERSION} +ARG OS_VARIANT +ENV OS_VARIANT=${OS_VARIANT} + +# This is required should be supplied as a build-arg +ARG WHEEL_TO_INSTALL +RUN test -n "${WHEEL_TO_INSTALL}" || (echo "Must supply WHEEL_TO_INSTALL as build arg"; exit 1) + +COPY dist/${WHEEL_TO_INSTALL} dist/${WHEEL_TO_INSTALL} + +# NodeJS needs libstdc++ to be present +# https://github.com/nodejs/unofficial-builds/#builds +RUN if echo "${OS_VARIANT}" | grep -e "alpine"; then \ + apk add libstdc++; \ + fi + +RUN pip install dist/${WHEEL_TO_INSTALL} + +RUN python -m nodejs --version +RUN python -m nodejs.npm --version From 9147ada7acf6b746684a9b573bfce98b25308622 Mon Sep 17 00:00:00 2001 From: Jacob Roberts Date: Mon, 7 Nov 2022 09:40:41 -0800 Subject: [PATCH 5/7] Add comment about where the sources of the bits come from for NodeJS releases --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 660dd90..f5c859f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,10 @@ Node.js PyPI distribution The [nodejs-bin][pypi] Python package redistributes Node.js so that it can be used as a dependency of Python projects. With `nodejs-bin` you can call `nodejs`, `npm` and `npx` from both the [command line](#command-line-usage) and a [Python API](#python-api-usage). -**Note: this is an unofficial Node.js distribution.** +**Note: this is an unofficial Node.js distribution.** However, it _does_ use only official bits distributed by the official NodeJS maintainers from one of the following sources: + +* NodeJS official releases: https://nodejs.org/en/download/releases/ +* NodeJS "unofficial" builds: https://github.com/nodejs/unofficial-builds/ **This is intended for use within Python virtual environments and containers, it should probably not be used for global installation.** From 490040e86fa4b668454c49d5055b47898a1318b4 Mon Sep 17 00:00:00 2001 From: Jacob Roberts Date: Mon, 7 Nov 2022 09:47:40 -0800 Subject: [PATCH 6/7] Update the names of jobs in the GH actions test workflow --- .github/workflows/test.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7de1123..15bbba0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -35,14 +35,14 @@ jobs: if-no-files-found: error retention-days: 1 - test: + test-non-linux: name: "Test ${{ matrix.os }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}" runs-on: ${{ matrix.os }} needs: [build-wheels] strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [windows-latest, macos-latest] nodejs-version: ['14.19.3', '16.15.1', '18.4.0'] python-version: ['3.7', '3.8', '3.9', '3.10'] @@ -79,7 +79,8 @@ jobs: run: python -W error -m nodejs --version python -W error -m nodejs.npm --version - test-docker: + + test-linux: name: "Test Docker OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}" runs-on: ubuntu-latest needs: [build-wheels] From 7aa076a3c485d6f36c1871fe34b2c6db59af982d Mon Sep 17 00:00:00 2001 From: Jacob Roberts Date: Mon, 7 Nov 2022 09:51:19 -0800 Subject: [PATCH 7/7] Update the sanity tests to include all of the subcommands exposed by this package --- .github/workflows/test.yaml | 2 ++ Dockerfile | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 15bbba0..fc863d6 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -79,6 +79,8 @@ jobs: run: python -W error -m nodejs --version python -W error -m nodejs.npm --version + python -W error -m nodejs.npx --version + python -W error -m nodejs.corepack --version test-linux: name: "Test Docker OS:${{ matrix.os-variant }} Python:${{ matrix.python-version }} NodeJS:${{ matrix.nodejs-version }}" diff --git a/Dockerfile b/Dockerfile index 97f7286..670ee07 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,5 +22,7 @@ RUN if echo "${OS_VARIANT}" | grep -e "alpine"; then \ RUN pip install dist/${WHEEL_TO_INSTALL} -RUN python -m nodejs --version -RUN python -m nodejs.npm --version +RUN python -W error -m nodejs --version +RUN python -W error -m nodejs.npm --version +RUN python -W error -m nodejs.npx --version +RUN python -W error -m nodejs.corepack --version