Extension deb builder (#303)

Author: levkk

pgml-extension/pgml_rust/.dockerignore

@@ -0,0 +1 @@
+target/

pgml-extension/pgml_rust/Dockerfile

@@ -0,0 +1,51 @@
+#
+# Extension builder.
+#
+ARG VERSION=22.04
+FROM ubuntu:${VERSION}
+
+ARG PGVERSION=14
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Etc/UTC
+
+# Apt-fast
+RUN apt-get update && apt-get install software-properties-common -y
+
+RUN add-apt-repository ppa:apt-fast/stable --yes
+RUN apt-get update && apt-get -y install apt-fast
+
+RUN  apt-fast install apt-fast curl unzip gpg -y
+
+# PostgresSQL
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg > /dev/null
+RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
+
+# CMake
+RUN curl -L https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+RUN echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null
+
+RUN apt-get update && apt-fast install postgresql-${PGVERSION} libopenblas-dev cmake postgresql-server-dev-${PGVERSION} pkg-config libssl-dev build-essential libclang-dev -y
+
+USER postgres
+
+# Install Rust
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+ENV PATH="/var/lib/postgresql/.cargo/bin:${PATH}"
+
+# Install tcdi/pgx
+RUN cargo install cargo-pgx
+RUN cargo pgx init --pg${PGVERSION} /usr/bin/pg_config
+
+COPY --chown=postgres:postgres . /app
+WORKDIR /app
+
+# Build and upload package to S3
+RUN cargo pgx package
+
+# Deb file goes here, mount it on your local system
+VOLUME /output
+USER root
+
+# Run
+ENTRYPOINT ["bash", "docker/build_ubuntu_deb.sh", "0.0.4", "/output"]
+

pgml-extension/pgml_rust/README.md

@@ -4,20 +4,36 @@ Here we have some POC code to use Rust for PostgresML.
 
 ## Dependencies
 
-All dependencies are vendored. I downloaded XGBoost 1.62 and all its submodules. We're also using the `master` branch of `xgboost` Rust crate.
+All dependencies are vendored. I downloaded XGBoost 1.62 and all its submodules. We're also using the `master` branch of `xgboost` Rust crate and `openblas-src`.
 
 If you haven't already, install:
 
 - `cmake`
 - `libclang-dev`
+- `libopenblas-dev`
 
 ## Local development
 
-1. `cargo install pgx`
-2. `cargo pgx run`
-3. `DROP EXTENSION IF EXISTS pgml_rust;`
-4. `CREATE EXTENSION pgml_rust;`
-5. `SELECT pgml_train('pgml.diabetes', ARRAY['age', 'sex'], 'target');`
-6. `SELECT * FROM pgml_predict(ARRAY[1, 5.0]);`
+1. `cargo install cargo-pgx`
+2. `cargo pgx init`
+3. `cargo pgx run`
+4. `DROP EXTENSION IF EXISTS pgml_rust;`
+5. `CREATE EXTENSION pgml_rust;`
+6. `SELECT pgml_rust.train('Project name', 'regression', pgml_rust.diabetes', 'target', 'xgboost', '{}');`
+7. `SELECT * FROM pgml_rust.predict('Project name', ARRAY[1, 5.0, 2.0]);`
 
-Lots of todos, but still a decent PoC.
+## Packaging
+
+We currently support Ubuntu 18.04 and newer. Mac OS (Apple Sillicon) support is in progress. Provided in the repo is the `.deb` builder, which requires Docker. Once Docker is installed, you can run:
+
+```bash
+bash build_extension.sh
+```
+
+which will produce a `.deb` file in the current directory. The deb file can be installed with `apt-get`, for example:
+
+```bash
+apt-get install ./postgresql-pgml-12_0.0.4-ubuntu20.04-amd64.deb
+```
+
+which will take care of installing its dependencies as well. Make sure to run this as root and not with sudo.

pgml-extension/pgml_rust/build_deb.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+# Build the extension.
+#
+
+echo "Building base image, this will take a little while"
+docker build . --build-arg VERSION=${1:-"22.04"} --build-arg PGVERSION=${2:-"14"}
+
+IMAGE_ID=$(docker images | awk '{print $3}' | awk 'NR==2')
+
+docker run -v $(pwd):/output ${IMAGE_ID}

pgml-extension/pgml_rust/build_extension.sh

@@ -1,9 +1,9 @@
-Package: postgresql-pgml-14
+Package: postgresql-pgml-PGVERSION
 Version: VERSION
 Section: base
 Priority: optional
 Architecture: ARCH
-Depends: postgresql-14
+Depends: postgresql-PGVERSION, libopenblas-dev, postgresql-server-dev-PGVERSION
 Maintainer: PostgresML <team@postgresml.org>
 Description: PostgresML - machine learning with PostgreSQL
  PostgresML is a PostgreSQL extension that allows to do machine

pgml-extension/pgml_rust/control

@@ -0,0 +1,49 @@
+#!/bin/bash
+#
+# Build a .deb for the Postgres and Ubuntu version.
+#
+
+VERSION=${1:-0.0.4}
+OUTPUT_DIR=${2:-"."}
+
+if [[ $(uname) == *"aarch64"* ]]; then
+    ARCH="arm64"
+else
+    ARCH="amd64"
+fi
+
+PGVERSION=$(pg_config | grep "VERSION")
+
+if [[ $PGVERSION == *"12."* ]]; then
+    PGVERSION="12"
+elif [[ $PGVERSION == *"13."* ]]; then
+    PGVERSION="13"
+elif [[ $PGVERSION == *"14."* ]]; then
+    PGVERSION="14"
+elif [[ $PGVERSION == *"11."* ]]; then
+    PGVERSION="11"
+elif [[ $PGVERSION == *"10."* ]]; then
+    PGVERSION="10"
+else
+    echo "Unknown PostgreSQL version detected: ${PGVERSION}"
+    exit 1
+fi
+
+TARGET="target/release/pgml_rust-pg${PGVERSION}"
+UBUNTU_VERSION=$(lsb_release -a | grep Release | awk '{ print $2 }')
+
+mkdir -p ${TARGET}/DEBIAN
+cp control ${TARGET}/DEBIAN/control
+
+# Save version and arch.
+sed -i "s/PGVERSION/${PGVERSION}/g" ${TARGET}/DEBIAN/control
+sed -i "s/VERSION/${VERSION}/g" ${TARGET}/DEBIAN/control
+sed -i "s/ARCH/${ARCH}/g" ${TARGET}/DEBIAN/control
+
+# Show me what we got.
+cat ${TARGET}/DEBIAN/control
+
+PACKAGE=postgresql-pgml-${PGVERSION}_${VERSION}-ubuntu${UBUNTU_VERSION}-${ARCH}.deb
+
+# Build the debian package
+dpkg-deb --build ${TARGET} $OUTPUT_DIR/${PACKAGE}