RFC: FPM packaging

.github/workflows/package-build.yaml

@@ -0,0 +1,65 @@
+# This is a reusable workflow, as defined by
+# https://docs.github.com/en/actions/using-workflows/reusing-workflows
+#
+# Maintain this separately as the "meat" of the packaging work.  Other
+# files of form packaging-*.yaml call this one from a different
+# perspective.
+#
+# To handle the documented limitations of reusable workflows, make
+# sure the near the /!\ comments match your intended repository and
+# user.
+
+name: Package for publication
+on:
+  workflow_call:
+    inputs:
+      fallback_version:
+        type: string
+        required: true
+      git_ref:
+        type: string
+        required: true
+      deb_version:
+        type: string
+        required: false
+      rpm_version:
+        type: string
+        required: false
+      pacman_tgz_version:
+        type: string
+        required: false
+defaults:
+  run:
+    shell: bash
+jobs:
+  package-pash:
+      runs-on: ubuntu-latest
+      steps:
+        - name: Clone PaSh
+          uses: actions/checkout@v2
+          with:
+            ref: ${{ inputs.git_ref }}
+        - name: Clone FPM
+          uses: actions/checkout@v2
+          with:
+            repository: jordansissel/fpm
+            token: ${{ secrets.GH_PAT }}
+            path: scripts/package/fpm
+        - name: Build packages
+          run: |
+            build() {
+              ./deploy.sh "$1" "${2:-${{ inputs.fallback_version }}}" "$3";
+            }
+            cd scripts/package
+            sed -i 's/--squash//g' fpm/Makefile
+            DOCKERHUB_TOKEN='${{ secrets.DOCKERHUB_TOKEN }}'
+            DOCKERHUB_USERNAME='${{ secrets.DOCKERHUB_USERNAME }}'
+            echo "$DOCKERHUB_TOKEN" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin;
+            build ubuntu '${{ inputs.deb_version }}' deb;
+            build fedora '${{ inputs.rpm_version }}' rpm;
+            build archlinux '${{ inputs.pacman_tgz_version }}' pacman;
+        - name: Upload Packages as Artifacts
+          uses: actions/upload-artifact@v3
+          with:
+            name: Packages
+            path: scripts/package/output

.github/workflows/package-gui.yaml

@@ -0,0 +1,37 @@
+# For use in GitHub's Actions GUI
+# https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/
+name: On-demand Packaging
+on:
+  workflow_dispatch:
+    branches:
+      - '*'
+    inputs:
+      fallback_version:
+        # TODO: The comment in ./package-pull-requests.yaml
+        # impacts whether this is good for the user to read.
+        description: 'What fallback version string can I use for some package format? Compatibility not guarenteed for all package formats.'
+        default: '0.0.1'
+        required: true
+      git_ref:
+        description: 'Git reference, like a branch name or commit hash'
+        default: 'main'
+        required: true
+      deb_version:
+        description: "What version string should I use in .deb format packages, if not the fallback?"
+        required: false
+      rpm_version:
+        description: '.rpm?'
+        required: false
+      pacman_tgz_version:
+        description: "pacman's .tar.gz?"
+        required: false
+jobs:
+  build-packages:
+    uses: zyrolasting/pash/.github/workflows/package-build.yaml@fpm
+    with:
+      fallback_version: ${{ inputs.fallback_version }}
+      deb_version: ${{ inputs.deb_version }}
+      rpm_version: ${{ inputs.rpm_version }}
+      pacman_tgz_version: ${{ inputs.pacman_tgz_version }}
+      git_ref: ${{ inputs.git_ref }}
+    secrets: inherit

.github/workflows/package-pull-requests.yaml

@@ -0,0 +1,17 @@
+name: Package Public Pull Requests (always 0.0.0)
+on:
+  pull_request:
+    branches:
+        - main
+        - future
+jobs:
+  build-packages:
+    if: ${{ github.event.pull_request.draft == false }}
+    uses: zyrolasting/pash/.github/workflows/package-build.yaml@main
+    with:
+      # TODO: consider using version files in the repository
+      # 0.0.0 has been tested to work in Fedora, Ubuntu, Debian, and Arch,
+      # which does not imply compatibility with all package formats.
+      fallback_version: '0.0.0'
+      git_ref: ${{ github.event.pull_request.head.sha }}
+    secrets: inherit

.github/workflows/package.yaml

@@ -0,0 +1,27 @@
+name: Packaging
+on: 
+  pull_request:
+    branches:
+        - main
+        - future
+
+jobs:
+  package-pash:
+      runs-on: ubuntu-latest
+      if: github.event.pull_request.draft == false
+      steps:
+        - name: Checkout code
+          uses: actions/checkout@v2 
+          with:
+            ref: ${{ github.event.pull_request.head.sha }}
+        - name: Make .deb
+          run: bash scripts/package/deploy.sh ubuntu "$GITHUB_HEAD_REF" deb
+        - name: Make .rpm
+          run: bash scripts/package/deploy.sh fedora "$GITHUB_HEAD_REF" rpm
+        - name: Make .tar.gz (Pacman)
+          run: bash scripts/package/deploy.sh archlinux "$GITHUB_HEAD_REF" pacman
+        - name: Upload Packages as Artifacts
+          uses: actions/upload-artifact@v3
+          with:
+            name: Packages
+            path: scripts/package/output

docs/tooling/packaging.md

@@ -0,0 +1,75 @@
+# Packaging PaSh
+
+`scripts/package` packages PaSh into various formats. The software inside is a suitable
+replacement for deployments depending on `install.sh`,
+`setup-pash.sh`, `up.sh`, `distro-deps.sh`, and/or `pkg.sh`.
+
+
+## Building Packages
+[eef]: https://github.com/docker/cli/blob/master/experimental/README.md
+
+```
+build.sh VERSION [OUTPUT_FORMAT ...]
+```
+
+Creates new package files using the repository's current content, each
+with mode 440. All output files appear in `scripts/package/output`.
+
+**Clean up junk files and new test results!** The scripts use a
+reasonable ignore list, but otherwise defaults to including files
+within the PaSh repository.
+
+Command-line arguments correspond to those found in [FPM's
+CLI](https://fpm.readthedocs.io/en/v1.14.2/cli-reference.html):
+
+  - `VERSION` is the value of `--version`.
+  - `OUTPUT_FORMAT` is some value for `--output-type`
+
+
+## Testing Packages
+
+```
+  deploy.sh IMAGE VERSION FORMAT
+```
+
+Performs a test deployment of PaSh, as follows:
+
+1. Run `build.sh VERSION FORMAT`
+2. Start Docker container based on `docker pull IMAGE`
+3. Install the package from Step 1 in the container from Step 2.
+4. Run the newly-installed `pa.sh` against a bundled `evaluation/intro/hello-world.sh` example.
+
+Here, `deploy.sh` installs a `pacman`/`.tar.gz` package deploys in
+Arch, a `.deb` package for Debian and Ubuntu, and an RPM package for
+Fedora.
+
+```
+$ version=0.0.1
+$ PATH="$PASH_TOP/scripts/package:$PATH"
+$ deploy.sh archlinux $version pacman
+$ deploy.sh debian $version deb
+$ deploy.sh ubuntu $version deb
+$ deploy.sh fedora $version rpm
+```
+
+## Developing Packages
+
+```
+  iterate.sh IMAGE FORMAT
+```
+
+Runs `deploy.sh IMAGE 0.0.1 FORMAT`, then prompts to repeat.
+
+
+## GitHub Actions
+
+The source code repository defines workflows for GitHub actions.  The
+workflows for packaging all use `deploy.sh` to build and run PaSh's
+"Hello, World" example. One workflow responds to new pull requests.
+
+**On-Demand Packaging** is the only workflow that can be directly
+used in the GitHub GUI. You can find it under the **Actions** tab.
+
+![image](https://user-images.githubusercontent.com/1312121/181671950-89ec5f57-5b9f-4fdb-90a2-6d1099257ad1.png)
+
+Click "Run Workflow" and fill out the form. The form defines what commit of PaSh gets packaged, and each package format may have its own version string.

pa.sh

@@ -2,9 +2,15 @@
 
 export PASH_TOP=${PASH_TOP:-${BASH_SOURCE%/*}}
 export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
-# point to the local downloaded folders
-export PYTHONPATH=${PASH_TOP}/python_pkgs/
-## Register the signal handlers, we can add more signals here
+
+# Until PaSh is set up in a way that naturally integrates with
+# Python's system-level paths, prefer local package installations.
+export PYTHONPATH="$(
+  find "$PASH_TOP" \
+       -type d \( -name site-packages -or -name dist-packages \) \
+       -printf '%p:'
+)$PASH_TOP/python_pkgs"
+
 trap kill_all SIGTERM SIGINT
 
 ## kill all the pending processes that are spawned by this shell
@@ -14,6 +20,7 @@ function kill_all() {
     # kill pash_daemon
     kill -s SIGKILL "$daemon_pid"
 }
+
 ## Save the umask to first create some files and then revert it
 old_umask=$(umask)
 

requirements.txt

@@ -0,0 +1,4 @@
+jsonpickle
+matplotlib
+numpy
+pexpect

scripts/package/.gitignore

@@ -0,0 +1,2 @@
+fpm
+output

scripts/package/build.sh

@@ -0,0 +1,45 @@
+#! /usr/bin/env bash
+# Launch REPL for building PaSh system packages.
+
+here="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+PASH_TOP=$(readlink -f "$here/../..")
+
+# We need FPM's source code to build a key Docker image.
+if [ ! -d "$here/fpm" ]; then
+    git clone --depth 1 git@github.com:jordansissel/fpm.git "$here/fpm"
+    sed -i 's/--squash//g' "$here/fpm/Makefile"
+fi
+
+# Make a Docker image that knows about building packages.
+if ! docker image inspect fpm 2>&1 >/dev/null; then
+    make -C "$here/fpm" docker-release-everything
+fi
+
+mkdir -p "$here/output"
+
+main() {
+    local docker_interactive_flags=''
+    local entrypoint_interactive_flags=''
+
+    if [[ $- == *i* ]]; then
+        set -x
+        docker_interactive_flags='--interactive --tty'
+        entrypoint_interactive_flags='--rcfile'
+        set +x
+    fi
+
+    docker run \
+	   --entrypoint /bin/bash \
+	   --rm \
+	   --user "$(id -u):$(id -g)" \
+	   --volume "$here/output:/out" \
+	   --volume "$here/tools:/tools" \
+	   --volume "$PASH_TOP:/src" \
+           $docker_interactive_flags \
+           fpm \
+           $entrypoint_interactive_flags \
+           /tools/start "$@"
+}
+
+# Only enter if script was not sourced
+(return 2>/dev/null) || main "$@"

scripts/package/deploy.sh

@@ -0,0 +1,61 @@
+#! /usr/bin/env bash
+
+# Build, install, and use PaSh in a new Docker container.
+#
+#   deploy.sh IMAGE VERSION FORMAT
+#
+# Exit code matches that from running `./tests/$3/main.sh` in the
+# container.  0 is a sign of PaSh's functionality in the other
+# environment.
+#
+# Scripts in ./tests may use /package in the container to store
+# information on the host, but printing should suffice.
+#
+# $1 - IMAGE name from Docker Hub to use
+# $2 - VERSION of PaSh to install in a container
+# $3 - FPM package FORMAT for the named PaSh version
+
+set -euo pipefail
+
+me="$(readlink -f "${BASH_SOURCE[0]}")"
+mydirname="$(dirname "$me")"
+myname="$(basename "$me")"
+
+if [ -f /.dockerenv ]; then
+    set +x
+    # Pass control to vertification script.
+    target_version="$1"
+    output_format="$2"
+
+    # Keep this here, so tests don't need to keep setting the variable.
+    export PASH_TOP=/usr/lib/pash
+
+    "$mydirname/fire-hook.sh" "before-test" "$output_format" "$target_version"
+
+    # Basic example must always work, no matter where we're going.
+    [ -d "$PASH_TOP" ]
+    test_script="${PASH_TOP}/evaluation/intro/hello-world.sh"
+    set -x
+    time pa.sh "$test_script"
+    time bash "$test_script"
+    set +x
+else
+    image="$1"
+    target_version="$2"
+    output_format="$3"
+
+    # Perform a one-off build for the package.
+    "$mydirname/build.sh" "$target_version" "$output_format"
+
+    # It's important to use Docker hub here because we want to verify
+    # PaSh's behavior against other people's work, namely official OS
+    # images.
+    docker pull "$image"
+
+    # Invoke self in the container.
+    docker run \
+	   --rm \
+	   --volume "${mydirname}:/package" \
+	   "$image" \
+	   "/package/${myname}" "$target_version" "$output_format"
+fi

scripts/package/fire-hook.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+# Fire hook script in response to an event. Hooks are format-specific,
+# so Debian packages can respond differently than Arch packages, for
+# example.
+
+set -eu
+event_name="$1"; shift
+
+here="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
+event="$here/hooks/${event_name}"
+
+if [ -f "$event/hook" ]; then
+    # Pass directory path so the script don't need to figure out where
+    # it is.  -f implies the script does not have to be Bash, but use
+    # of non-executables must raise an error.
+    "$event/hook" "$event" "$@"
+fi