WIP gdal/proj/geos updates

.github/workflows/build_package.yml

@@ -0,0 +1,58 @@
+name: Build and upload package
+run-name: "Build and upload ${{ inputs.package }} ${{ inputs.version }}${{ inputs.dry_run && ' (dry run)' || '' }}"
+
+on:
+  workflow_dispatch:
+    inputs:
+      package:
+        description: "Package to build"
+        type: choice
+        options:
+          - GDAL
+          - GEOS
+          - PROJ
+        required: true
+      version:
+        description: "Version of the package (X.Y.Z)"
+        type: string
+        required: true
+      dry_run:
+        description: "Skip uploading to S3 (dry run)"
+        type: boolean
+        default: true # TODO
+        required: false
+
+permissions:
+  contents: read
+
+env:
+  # AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+  AWS_DEFAULT_REGION: "us-east-1"
+  S3_BUCKET: "heroku-buildpack-geo"
+
+jobs:
+  build:
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        stack_version: ["20", "22", "24"]
+        runner: ["ubuntu-latest", "pub-hk-ubuntu-22.04-xlarge", "pub-hk-ubuntu-22.04-large"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build Docker image
+        run: make buildenv STACK_VERSION=${{ matrix.stack_version }}
+      - name: Compile and package ${{ inputs.package }} ${{ inputs.version }}
+        run: |
+          BUILD_SCRIPT="./$(echo "${{ inputs.package }}" | tr '[:upper:]' '[:lower:]').sh"
+          docker run --rm -v "${PWD}/upload:/tmp/upload" geo-buildenv-${{ matrix.stack_version }} "${BUILD_SCRIPT}" '${{ inputs.version }}'
+      - name: Test package
+        run: |
+          RUN_IMAGE='heroku/heroku:${{ matrix.stack_version }}'
+          ARCHIVE_FILEPATH='/upload/heroku-${{ matrix.stack_version }}/${{ inputs.package }}/${{ inputs.package }}-${{ inputs.version }}.tar.gz'
+          docker run --rm -v "${PWD}/upload:/upload:ro" -v "${PWD}/builds:/builds:ro" "${RUN_IMAGE}" /builds/test_package.sh "${ARCHIVE_FILEPATH}"
+      - name: Upload package to S3
+        if: (!inputs.dry_run)
+        run: aws s3 sync ./upload "s3://${S3_BUCKET}"

.github/workflows/ci.yml

@@ -16,8 +16,8 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - run: shellcheck -x bin/*
-      - run: find . -name "*.sh" -exec shellcheck -x {} \;
+      - name: Run ShellCheck
+        run: shellcheck --check-sourced --color=always bin/* builds/*.sh tests.sh
 
   test:
     runs-on: ubuntu-latest
@@ -28,4 +28,5 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - run: make test STACK_VERSION='${{ matrix.stack_version }}'
+      - name: Run tests
+        run: make test STACK_VERSION='${{ matrix.stack_version }}'

.gitignore

@@ -0,0 +1,2 @@
+upload/
+.DS_Store

Makefile

@@ -1,7 +1,7 @@
 # These targets are not files
-.PHONY: compile test build-heroku-20 build-heroku-22
+.PHONY: compile test buildenv test-archive
 
-STACK_VERSION ?= 22
+STACK_VERSION ?= 24
 STACK := heroku-$(STACK_VERSION)
 BASE_BUILD_IMAGE := heroku/heroku:$(STACK_VERSION)-build
 PLATFORM := linux/amd64
@@ -10,39 +10,26 @@ compile:
 	@echo "Running compile using: STACK_VERSION=$(STACK_VERSION)"
 	@echo "To use a different stack, run: 'make compile STACK_VERSION=NN'"
 	@echo
-	@docker run --rm --platform=$(PLATFORM) -v "$(PWD):/src:ro" -e "STACK=$(STACK)" -w /buildpack "$(BASE_BUILD_IMAGE)" \
-		bash -c 'cp -r /src/bin /buildpack && mkdir -p /tmp/{build,cache,env} && bin/compile /tmp/build /tmp/cache /tmp/env'
+	@docker run --rm --platform=$(PLATFORM) -v "$(PWD)/bin:/tmp/buildpack/bin:ro" -e "STACK=$(STACK)" -w /tmp/buildpack "$(BASE_BUILD_IMAGE)" \
+		bash -c 'mkdir -p /tmp/{build,cache,env} && bin/compile /tmp/build /tmp/cache /tmp/env'
 	@echo
 
 test:
 	@echo "Running tests using: STACK_VERSION=$(STACK_VERSION)"
 	@echo "To use a different stack, run: 'make test STACK_VERSION=NN'"
 	@echo
-	@docker run --rm --platform=$(PLATFORM) -v "$(PWD):/buildpack:ro" -e "STACK=$(STACK)" "$(BASE_BUILD_IMAGE)" /buildpack/tests.sh
+	@docker run --rm --platform=$(PLATFORM) --user root -v "$(PWD):/buildpack:ro" -e "STACK=$(STACK)" "$(BASE_BUILD_IMAGE)" /buildpack/tests.sh
 	@echo
 
-build-heroku-20:
-	@echo "Creating build environment (heroku-20)..."
+buildenv:
+	@echo "Creating build environment using: STACK_VERSION=$(STACK_VERSION)"
+	@echo "To use a different stack, run: 'make buildenv STACK_VERSION=NN'"
 	@echo
-	@docker build --pull -f "$(shell pwd)/builds/Dockerfile-heroku-20" -t buildenv-heroku-20 .
+	@docker build --pull --platform=$(PLATFORM) --build-arg="STACK_VERSION=$(STACK_VERSION)" -t "geo-buildenv-$(STACK_VERSION)" ./builds/
 	@echo
 	@echo "Usage..."
 	@echo
-	@echo "  $$ export S3_BUCKET='heroku-geo-buildpack' # Optional unless deploying"
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ ./builds/gdal/gdal-<version>.sh"
+	@echo '  $$ docker run --rm -it -v "$${PWD}/upload:/tmp/upload" geo-buildenv-$(STACK_VERSION) ./proj.sh X.Y.Z'
+	@echo '  $$ docker run --rm -it -v "$${PWD}/upload:/tmp/upload" geo-buildenv-$(STACK_VERSION) ./geos.sh X.Y.Z'
+	@echo '  $$ docker run --rm -it -v "$${PWD}/upload:/tmp/upload" geo-buildenv-$(STACK_VERSION) ./gdal.sh X.Y.Z'
 	@echo
-	@docker run -e STACK="heroku-20" -e S3_BUCKET -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -it --rm buildenv-heroku-20
-
-build-heroku-22:
-	@echo "Creating build environment (heroku-22)..."
-	@echo
-	@docker build --pull -f "$(shell pwd)/builds/Dockerfile-heroku-22" -t buildenv-heroku-22 .
-	@echo
-	@echo "Usage..."
-	@echo
-	@echo "  $$ export S3_BUCKET='heroku-geo-buildpack' # Optional unless deploying"
-	@echo "  $$ export AWS_ACCESS_KEY_ID=foo AWS_SECRET_ACCESS_KEY=bar  # Optional unless deploying"
-	@echo "  $$ ./builds/gdal/gdal-<version>.sh"
-	@echo
-	@docker run -e STACK="heroku-22" -e S3_BUCKET -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -it --rm buildenv-heroku-22

README.md

@@ -2,7 +2,7 @@ Heroku Buildpack: Geo
 =====================
 
 Heroku Buildpack Geo is a [Heroku buildpack](http://devcenter.heroku.com/articles/buildpacks) that
-installs the Geo/GIS libraries [GDAL](https://gdal.org/), [GEOS](https://trac.osgeo.org/geos/) and [PROJ](https://proj.org/)
+installs the Geo/GIS libraries [GDAL](https://gdal.org/), [GEOS](https://libgeos.org/) and [PROJ](https://proj.org/)
 
 It can be used to get [GeoDjango](https://docs.djangoproject.com/en/stable/ref/contrib/gis/) or [RGeo](https://github.com/rgeo/rgeo) running on Heroku.
 
@@ -25,9 +25,9 @@ Default Versions
 
 The buildpack will install the following versions by default *for new apps*:
 
-- GDAL: `3.5.0`
-- GEOS: `3.10.2`
-- PROJ: `8.2.1`
+- GDAL: `TODO`
+- GEOS: `TODO`
+- PROJ: `9.4.0`
 
 Note: *Existing apps* that don't specify an explicit version will continue to use the
 version used by the last successful build (unless the
@@ -40,10 +40,25 @@ Available Versions
 ------------------
 
 - GDAL:
-  - Heroku-20: `2.4.0`, `2.4.2`, `3.5.0`
-  - Heroku-22: `3.5.0`
-- GEOS: `3.7.2`, `3.10.2`
-- PROJ: `5.2.0`, `8.2.1`
+  - `2.4.0` (on Heroku-20 only)
+  - `2.4.2` (on Heroku-20 only)
+  - `3.5.0` (on Heroku-20 and Heroku-22 only)
+  - `TODO`
+  - `3.5.3`
+  - `3.6.4`
+  - `3.7.3`
+  - `3.8.5`
+  - `3.9.0`
+- GEOS:
+  - `3.7.2` (on Heroku-20 and Heroku-22 only)
+  - `3.9.5`
+  - `3.10.6`
+  - `3.11.3`
+  - `3.12.1`
+- PROJ:
+  - `5.2.0` (on Heroku-20 and Heroku-22 only)
+  - `8.2.1` (on Heroku-20 and Heroku-22 only)
+  - `9.4.0`
 
 Migrating from heroku/python GEO support
 ----------------------------------------

bin/compile

@@ -6,7 +6,7 @@ vendor_lib() {
     VENDOR_DIR=$3
 
     echo "-----> Installing $LIBRARY-$VERSION"
-    LIBRARY_URL="https://heroku-buildpack-geo.s3.amazonaws.com/${STACK}/${LIBRARY}/${LIBRARY}-${VERSION}.tar.gz"
+    LIBRARY_URL="https://heroku-buildpack-geo.s3.us-east-1.amazonaws.com/${STACK}/${LIBRARY}/${LIBRARY}-${VERSION}.tar.gz"
 
     mkdir -p "$VENDOR_DIR"
     if ! curl -sSf --retry 3 --retry-connrefused --connect-timeout 10 "${LIBRARY_URL}" | tar -zx -C "${VENDOR_DIR}"; then

builds/Dockerfile

@@ -0,0 +1,19 @@
+ARG STACK_VERSION="24"
+FROM heroku/heroku:${STACK_VERSION}-build
+
+ARG STACK_VERSION
+ENV STACK="heroku-${STACK_VERSION}"
+
+# For Heroku-24 and newer, the build image sets a non-root default `USER`.
+USER root
+
+RUN apt-get update --error-on=any \
+    && apt-get install -y --no-install-recommends \
+      libkml-dev \
+      libsqlite3-dev \
+      sqlite3 \
+    && rm -rf /var/lib/apt/lists/*
+
+ADD . /heroku-geo-buildpack
+
+WORKDIR "/heroku-geo-buildpack"

builds/gdal.sh

@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# shellcheck source=builds/utils.sh
+source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
+
+VERSION="${1:?"Error: The GDAL version to build must be specified as the first argument."}"
+
+SRC_DIR="$(mktemp -d)"
+INSTALL_DIR="$(mktemp -d)"
+ARCHIVE_DIR="/tmp/upload/${STACK}/GDAL"
+
+CONCURRENCY="$(nproc)"
+
+echo "Building GDAL ${VERSION} for ${STACK}..."
+
+set -o xtrace
+
+# GDAL 3.0.0+ requires PROJ at build time.
+# vendor_dependency "PROJ" "9.4.0" "${INSTALL_DIR}"
+vendor_dependency "PROJ" "8.2.1" "${INSTALL_DIR}"
+# tar -zxf "/tmp/upload/${STACK}/PROJ/PROJ-9.4.0.tar.gz" -C "${INSTALL_DIR}"
+
+#TODO
+# libkmlbase.so.1 => not found
+# libkmldom.so.1 => not found
+# libkmlengine.so.1 => not found
+
+cd "${SRC_DIR}"
+curl -sSf --retry 3 --retry-connrefused --connect-timeout 10 --max-time 60 "https://download.osgeo.org/gdal/${VERSION}/gdal-${VERSION}.tar.gz" \
+  | tar --gzip --extract --strip-components=1 --directory .
+
+# https://gdal.org/development/building_from_source.html
+mkdir build
+cd build
+cmake \
+  -DBUILD_TESTING=OFF \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}" \
+  ..
+cmake --build . -j "${CONCURRENCY}"
+cmake --build . --target install -j "${CONCURRENCY}"
+
+# gdal-config has to be skipped since whilst it's executable it's a bash script and not a binary that can be stripped.
+find "${INSTALL_DIR}" -type f \( -executable -o -name '*.so*' \) ! -name 'gdal-config' -print -exec strip --strip-unneeded '{}' +
+
+mkdir -p "${ARCHIVE_DIR}"
+TAR_FILEPATH="${ARCHIVE_DIR}/GDAL-${VERSION}.tar"
+tar --create --format=pax --sort=name --file="${TAR_FILEPATH}" --directory="${INSTALL_DIR}" .
+gzip --best "${TAR_FILEPATH}"
+
+# TODO
+# - Start GHA workflow for generating binaries
+# - Check perms on S3 bucket given HTTP 403s
+# - Create IAM user + set creds on GitHub repo
+# -
+# -
+
+# 5.269 The following NEW packages will be installed:
+# 5.269   libboost-dev libboost1.83-dev libkml-dev libkmlbase1t64
+# 5.269   libkmlconvenience1t64 libkmldom1t64 libkmlengine1t64 libkmlregionator1t64
+# 5.269   libkmlxsd1t64 libminizip-dev libminizip1t64 libsqlite3-dev liburiparser-dev
+# 5.269   liburiparser1 sqlite3
+# 5.502 0 upgraded, 15 newly installed, 0 to remove and 0 not upgraded.
+# 5.502 Need to get 12.9 MB of archives.
+# 5.502 After this operation, 171 MB of additional disk space will be used.
+# 5.502 Get:1 http://archive.ubuntu.com/ubuntu noble/main amd64 libboost1.83-dev amd64 1.83.0-2.1ubuntu3 [10.7 MB]
+# 6.415 Get:2 http://archive.ubuntu.com/ubuntu noble/main amd64 libboost-dev amd64 1.83.0.1ubuntu2 [4308 B]
+# 6.416 Get:3 http://archive.ubuntu.com/ubuntu noble/universe amd64 libminizip1t64 amd64 1:1.3.dfsg-3.1ubuntu2 [22.2 kB]
+# 6.417 Get:4 http://archive.ubuntu.com/ubuntu noble/universe amd64 liburiparser1 amd64 0.9.7+dfsg-2build1 [35.8 kB]
+# 6.419 Get:5 http://archive.ubuntu.com/ubuntu noble/universe amd64 libkmlbase1t64 amd64 1.3.0-12build1 [49.9 kB]
+# 6.420 Get:6 http://archive.ubuntu.com/ubuntu noble/universe amd64 libkmldom1t64 amd64 1.3.0-12build1 [156 kB]
+# 6.428 Get:7 http://archive.ubuntu.com/ubuntu noble/universe amd64 libkmlengine1t64 amd64 1.3.0-12build1 [71.4 kB]
+# 6.430 Get:8 http://archive.ubuntu.com/ubuntu noble/universe amd64 libkmlconvenience1t64 amd64 1.3.0-12build1 [49.8 kB]
+# 6.431 Get:9 http://archive.ubuntu.com/ubuntu noble/universe amd64 libkmlregionator1t64 amd64 1.3.0-12build1 [21.4 kB]
+# 6.432 Get:10 http://archive.ubuntu.com/ubuntu noble/universe amd64 libkmlxsd1t64 amd64 1.3.0-12build1 [29.7 kB]
+# 6.433 Get:11 http://archive.ubuntu.com/ubuntu noble/universe amd64 libminizip-dev amd64 1:1.3.dfsg-3.1ubuntu2 [30.1 kB]
+# 6.511 Get:12 http://archive.ubuntu.com/ubuntu noble/main amd64 libsqlite3-dev amd64 3.45.1-1ubuntu2 [911 kB]
+# 6.529 Get:13 http://archive.ubuntu.com/ubuntu noble/universe amd64 liburiparser-dev amd64 0.9.7+dfsg-2build1 [15.0 kB]
+# 6.530 Get:14 http://archive.ubuntu.com/ubuntu noble/main amd64 sqlite3 amd64 3.45.1-1ubuntu2 [144 kB]
+# 6.533 Get:15 http://archive.ubuntu.com/ubuntu noble/universe amd64 libkml-dev amd64 1.3.0-12build1 [659 kB]