Migrate Python runtime builds to GitHub Actions (#1432)

* Builds are now performed on GitHub Actions (which supports IP allowlisting),
   and can be triggered via the dispatch job workflow.
* The build process no longer uses the legacy bob-builder tool.
* The identical and thus redundant per-version scripts have been removed.
* The per-stack Dockerfiles have been consolidated.

GUS-W-12964868.

.github/dependabot.yml

@@ -1,9 +1,5 @@
 version: 2
 updates:
-  - package-ecosystem: "pip"
-    directory: "/"
-    schedule:
-      interval: "monthly"
   - package-ecosystem: "bundler"
     directory: "/"
     schedule:

.github/workflows/build_python_runtime.yml

@@ -0,0 +1,74 @@
+name: Build and upload Python runtime
+run-name: "Build and upload Python ${{ inputs.python_version }}${{ inputs.dry_run && ' (dry run)' || '' }}"
+
+on:
+  workflow_dispatch:
+    inputs:
+      python_version:
+        description: "The Python version to build, specified as X.Y.Z"
+        type: string
+        required: true
+      dry_run:
+        description: "Skip deploying to S3 (e.g. for testing)"
+        type: boolean
+        default: false
+        required: false
+
+permissions:
+  contents: read
+
+env:
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: "us-west-2"
+  S3_BUCKET: "heroku-buildpack-python"
+
+# Unfortunately these jobs cannot be easily written as a matrix since `matrix.exclude` does not
+# support expression syntax, and the `inputs` context is not available inside the job `if` key.
+jobs:
+  build-and-upload-heroku-18:
+    runs-on: pub-hk-ubuntu-22.04-xlarge
+    env:
+      STACK_VERSION: "18"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build Docker image
+        run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/
+      - name: Build and package Python runtime
+        run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+      - name: Upload Python runtime archive to S3
+        if: (!inputs.dry_run)
+        run: aws s3 sync ./upload "s3://${S3_BUCKET}"
+
+  build-and-upload-heroku-20:
+    runs-on: pub-hk-ubuntu-22.04-xlarge
+    env:
+      STACK_VERSION: "20"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build Docker image
+        run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/
+      - name: Build and package Python runtime
+        run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+      - name: Upload Python runtime archive to S3
+        if: (!inputs.dry_run)
+        run: aws s3 sync ./upload "s3://${S3_BUCKET}"
+
+  build-and-upload-heroku-22:
+    # We only support Python 3.9+ on Heroku-22.
+    if: (!startsWith(inputs.python_version, '3.7.') && !startsWith(inputs.python_version,'3.8.'))
+    runs-on: pub-hk-ubuntu-22.04-xlarge
+    env:
+      STACK_VERSION: "22"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build Docker image
+        run: docker build --pull --tag buildenv --build-arg=STACK_VERSION builds/
+      - name: Build and package Python runtime
+        run: docker run --rm --platform="linux/amd64" --volume="${PWD}/upload:/tmp/upload" buildenv ./build_python_runtime.sh "${{ inputs.python_version }}"
+      - name: Upload Python runtime archive to S3
+        if: (!inputs.dry_run)
+        run: aws s3 sync ./upload "s3://${S3_BUCKET}"

.gitignore

@@ -1,14 +1,4 @@
-*.pyc
-site
-.DS_Store
-
-/.envrc
+__pycache__/
 .hatchet/repos/
-
-#Venv
-buildpack/*
-
-builds/dockerenv.staging*
-builds/dockerenv.production
-
+.DS_Store
 .rspec_status

Makefile

@@ -1,12 +1,9 @@
 # These targets are not files
-.PHONY: lint lint-scripts lint-ruby compile builder-image buildenv deploy-runtimes publish
+.PHONY: lint lint-scripts lint-ruby compile publish
 
 STACK ?= heroku-22
-STACKS ?= heroku-18 heroku-20 heroku-22
 PLATFORM := linux/amd64
 FIXTURE ?= spec/fixtures/python_version_unspecified
-ENV_FILE ?= builds/dockerenv.default
-BUILDER_IMAGE_PREFIX := heroku-python-build
 
 # Converts a stack name of `heroku-NN` to its build Docker image tag of `heroku/heroku:NN-build`.
 STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
@@ -17,7 +14,7 @@ lint-scripts:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
 	@shellcheck -x bin/steps/collectstatic bin/steps/nltk bin/steps/pip-install bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
-	@shellcheck -x builds/runtimes/*
+	@shellcheck -x builds/*.sh
 
 lint-ruby:
 	@bundle exec rubocop
@@ -29,36 +26,5 @@ compile:
 		bash -c 'cp -r /src/{bin,vendor} /buildpack && cp -r /src/$(FIXTURE) /build && mkdir /cache /env && bin/compile /build /cache /env'
 	@echo
 
-builder-image:
-	@echo "Generating binary builder image for $(STACK)..."
-	@echo
-	@docker build --pull -f builds/$(STACK).Dockerfile --platform="$(PLATFORM)" -t "$(BUILDER_IMAGE_PREFIX)-$(STACK)" .
-	@echo
-
-buildenv: builder-image
-	@echo "Starting build environment for $(STACK)..."
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ bob build runtimes/python-X.Y.Z"
-	@echo
-	@docker run --rm -it --env-file="$(ENV_FILE)" -v $(PWD)/builds:/app/builds --platform="$(PLATFORM)" "$(BUILDER_IMAGE_PREFIX)-$(STACK)" bash
-
-deploy-runtimes:
-ifndef RUNTIMES
-	$(error No runtimes specified! Use: "make deploy-runtimes RUNTIMES='python-X.Y.Z ...' [STACKS='heroku-18 ...'] [ENV_FILE=...]")
-endif
-	@echo "Using: RUNTIMES='$(RUNTIMES)' STACKS='$(STACKS)' ENV_FILE='$(ENV_FILE)'"
-	@echo
-	@set -eu; for stack in $(STACKS); do \
-		$(MAKE) builder-image STACK=$${stack}; \
-		for runtime in $(RUNTIMES); do \
-			echo "Generating/deploying $${runtime} for $${stack}..."; \
-			echo; \
-			docker run --rm -it --env-file="$(ENV_FILE)" --platform="$(PLATFORM)" "$(BUILDER_IMAGE_PREFIX)-$${stack}" bob deploy "runtimes/$${runtime}"; \
-			echo; \
-		done; \
-	done
-
 publish:
 	@etc/publish.sh

builds/Dockerfile

@@ -0,0 +1,13 @@
+ARG STACK_VERSION="22"
+FROM --platform=linux/amd64 heroku/heroku:${STACK_VERSION}-build
+
+ARG STACK_VERSION
+ENV STACK="heroku-${STACK_VERSION}"
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+      libsqlite3-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /tmp
+COPY build_python_runtime.sh .

builds/README.md

@@ -1,80 +1,11 @@
 # Python Buildpack Binaries
 
-The binaries for this buildpack are built in Docker containers based on the Heroku stack image.
+The binaries for this buildpack are built on GitHub Actions, inside Docker containers based on the Heroku stack image.
 
-## Configuration
+Users with suitable repository access can trigger builds by:
 
-In order to publish binaries AWS credentials must be passed to the build container.
-If you are testing only the build (ie: `bob build`), these are optional.
-
-In addition, unless you are building the official binaries for Heroku (which use the defaults
-specified in each `Dockerfile`), you will need to override `S3_BUCKET` and `S3_PREFIX` to
-match your own S3 bucket/use case.
-
-If you only need to set AWS credentials, you can do so by setting the environment variables
-`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` before calling the make commands.
-
-For example:
-
-```bash
-set +o history # Disable bash history
-export AWS_ACCESS_KEY_ID=...
-export AWS_SECRET_ACCESS_KEY=...
-set -o history # Re-enable bash history
-make ...
-```
-
-If you need to override the default S3 bucket, or would prefer not to use credentials via
-environment variables, then you need to instead use a Docker env file like so:
-
-1. Copy the `builds/dockerenv.default` env file to a location outside the buildpack repository.
-2. Edit the new file, adding at a minimum the values for the variables
-   `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (see Docker
-   [env-file documentation](https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file)).
-3. Pass the path of the file to the make commands using `ENV_FILE`. For example:
-
-   ```bash
-   make ... ENV_FILE=~/.dockerenv.python-buildpack
-   ```
-
-## Launching an interactive build environment
-
-To start an interactive version of the build environment (ideal for development) use the
-`buildenv` make target, passing in the desired `STACK` name. For example:
-
-```bash
-make buildenv STACK=heroku-18
-```
-
-This will create the builder docker image based on the latest image for that stack, and
-then start a bash shell where you can run `bob build`, `bob deploy`, and so forth.
-
-The `builds/` directory is bind-mounted into the running container, so local build formula
-changes will appear there immediately without the need to rebuild the image.
-
-## Bulk deploying runtimes
-
-When a new Python version is released, binaries have to be generated for multiple stacks.
-To automate this, use the `deploy-runtimes` make target, which will ensure the builder
-image is up to date, and then run `bob deploy` for each runtime-stack combination.
-
-The build formula name(s) are passed using `RUNTIMES`, like so:
-
-```bash
-make deploy-runtimes RUNTIMES='python-X.Y.Z'
-```
-
-By default this will deploy to all supported stacks (see `STACKS` in `Makefile`),
-but this can be overridden using `STACKS`:
-
-```bash
-make deploy-runtimes RUNTIMES='python-X.Y.Z' STACKS='heroku-18'
-```
-
-Multiple runtimes can also be specified (useful for when adding a new stack), like so:
-
-```bash
-make deploy-runtimes RUNTIMES='python-A.B.C python-X.Y.Z' STACKS='heroku-22'
-```
-
-Note: Both `RUNTIMES` and `STACKS` are space delimited.
+1. Navigating to the [Build and upload Python runtime](https://github.com/heroku/heroku-buildpack-python/actions/workflows/build_python_runtime.yml) workflow.
+2. Opening the "Run workflow" prompt.
+3. Entering the desired Python version.
+4. Optionally checking the "Skip deploying" checkbox (if testing)
+5. Clicking "Run workflow".