Sbcl librarian runtime

.github/workflows/build.yml

@@ -4,104 +4,145 @@ on:
   pull_request:
   workflow_call:
 
+env:
+  # SBCL is built from source for its linkable runtime; see the build steps.
+  SBCL_VERSION: "2.6.7"
+  # Host used to cross-compile SBCL_VERSION on Linux. It cannot be apt's (22.04
+  # ships 2.1.11, too old) nor SBCL_VERSION's own binary release, which is linked
+  # against glibc 2.38 and will not run on 22.04. 2.5.0's binary needs only glibc
+  # 2.34 and is recent enough to build 2.6.7.
+  SBCL_BOOTSTRAP_VERSION: "2.5.0"
+  # libquil depends on three sbcl-librarian fixes (secondary-system FASL bundles,
+  # library prefix, configurable core name). They live on a branch of Rigetti's
+  # fork; move this back to quil-lang/sbcl-librarian once they are upstream.
+  SBCL_LIBRARIAN_REPO: "https://github.com/rigetti/sbcl-librarian.git"
+  SBCL_LIBRARIAN_REF: "fix-secondary-system-bundles"
+  # The Quicklisp dist supplying the Lisp dependencies. It has to be recent enough
+  # for current cl-quil: the long-standing 2022-04-01 pin predates clos-encounters
+  # and fails with SYSTEM-NOT-FOUND.
+  QUICKLISP_VERSION: "2026-01-01"
+
 jobs:  
   build-linux:
     name: Build libquil.so
-    runs-on: ubuntu-latest
+    # Pinned rather than ubuntu-latest: the artifact links libsbcl, so it inherits
+    # the glibc of whatever built it, and a build on 24.04 fails to load on 22.04
+    # with "version `GLIBC_2.38' not found".
+    runs-on: ubuntu-22.04
 
     steps:
       - name: root suid tar
         run: sudo chown root /bin/tar && sudo chmod u+s /bin/tar
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           path: "libquil"
 
       - name: Cache SBCL
         id: cache-sbcl
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             /usr/local/lib/libsbcl.so
             /usr/local/lib/sbcl
             /usr/local/bin/sbcl
             ~/quicklisp
             ~/.sbclrc
-          key: ${{ runner.os }}-build-${{ hashFiles('**/versions') }}
+          key: ${{ env.ImageOS }}-${{ runner.arch }}-build-${{ hashFiles('**/versions') }}
 
       - if: steps.cache-sbcl.outputs.cache-hit != 'true'
         name: Install SBCL, libraries, and quicklisp
         run: |
-          sudo apt update && sudo apt install -y sbcl build-essential
-          sudo git clone --single-branch --branch sbcl-2.2.4 https://git.code.sf.net/p/sbcl/sbcl /usr/src/sbcl
-          cd /usr/src/sbcl && sudo sh make.sh && sudo sh make-shared-library.sh
-          sudo apt remove -y sbcl
-          sudo sh install.sh 
+          sudo apt update && sudo apt install -y build-essential bzip2 wget
+          # SBCL is built from source because a linkable runtime (libsbcl.so) is
+          # required and neither make.sh nor any package manager produces one.
+          #
+          # The host is a binary release of SBCL_BOOTSTRAP_VERSION rather than apt's:
+          # cross compiling a current SBCL needs a host of roughly the same vintage,
+          # and 22.04 ships 2.1.11, which fails in make-host-1 with "FAILURE-P was set
+          # when creating genesis.fasl". 22.04 is used deliberately, since the
+          # artifact inherits the glibc of whatever builds it.
+          wget -q "https://downloads.sourceforge.net/project/sbcl/sbcl/$SBCL_BOOTSTRAP_VERSION/sbcl-$SBCL_BOOTSTRAP_VERSION-x86-64-linux-binary.tar.bz2" -O /tmp/sbcl-bin.tar.bz2
+          mkdir -p /tmp/sbcl-bin && tar xf /tmp/sbcl-bin.tar.bz2 -C /tmp/sbcl-bin --strip-components=1
+          cd /tmp/sbcl-bin && sudo sh install.sh
+          sudo git clone --single-branch --branch sbcl-$SBCL_VERSION https://git.code.sf.net/p/sbcl/sbcl /usr/src/sbcl
+          cd /usr/src/sbcl && sudo sh make.sh --with-sb-linkable-runtime && sudo sh make-shared-library.sh
+          sudo sh install.sh
+          sudo mkdir -p /usr/local/lib
           sudo cp src/runtime/libsbcl.so /usr/local/lib/libsbcl.so
 
           sudo apt install -y wget
-          wget -P /tmp/ 'https://beta.quicklisp.org/quicklisp.lisp'
-          sbcl --noinform --non-interactive --load /tmp/quicklisp.lisp --eval "(quicklisp-quickstart:install :dist-url \"http://beta.quicklisp.org/dist/quicklisp/2022-04-01/distinfo.txt\")"
-          sbcl --noinform --non-interactive --load ~/quicklisp/setup.lisp --eval '(ql-util:without-prompting (ql:add-to-init-file))'
+          if [ ! -f ~/quicklisp/setup.lisp ]; then
+            wget -P /tmp/ 'https://beta.quicklisp.org/quicklisp.lisp'
+            sbcl --noinform --non-interactive --load /tmp/quicklisp.lisp --eval "(quicklisp-quickstart:install :dist-url \"http://beta.quicklisp.org/dist/quicklisp/$QUICKLISP_VERSION/distinfo.txt\")"
+            sbcl --noinform --non-interactive --load ~/quicklisp/setup.lisp --eval '(ql-util:without-prompting (ql:add-to-init-file))'
+          fi
           echo "#+quicklisp(push (truename \"$GITHUB_WORKSPACE\") ql:*local-project-directories*)" >> ~/.sbclrc
           rm -f /tmp/quicklisp.lisp
           cat ~/.sbclrc
 
       - name: Pull Lisp dependencies
         run: |
-          git clone https://github.com/notmgsk/quilc.git $GITHUB_WORKSPACE/quilc && cd $GITHUB_WORKSPACE/quilc && git checkout bffea7fdb972cc9b3a50b790246b6a3143c88c7a
+          git clone https://github.com/quil-lang/quilc.git $GITHUB_WORKSPACE/quilc && cd $GITHUB_WORKSPACE/quilc && git checkout 3d83b665b3addc33be2f696b1cdebc0c4c5a04f9
           git clone https://github.com/quil-lang/qvm.git $GITHUB_WORKSPACE/qvm && cd $GITHUB_WORKSPACE/qvm && git checkout 4617625cb6053b1adfd3f7aea9cd2be328b225f6
           git clone https://github.com/quil-lang/magicl.git $GITHUB_WORKSPACE/magicl
           git clone https://github.com/stylewarning/cl-permutation $GITHUB_WORKSPACE/cl-permutation
-          git clone https://github.com/quil-lang/sbcl-librarian.git $GITHUB_WORKSPACE/sbcl-librarian && cd $GITHUB_WORKSPACE/sbcl-librarian && git checkout 04f7e390c777084ac43b8df9e90593a041da2381
+          git clone $SBCL_LIBRARIAN_REPO $GITHUB_WORKSPACE/sbcl-librarian && cd $GITHUB_WORKSPACE/sbcl-librarian && git checkout $SBCL_LIBRARIAN_REF
 
       - name: Install quilc dependencies
-        run: sudo apt update && sudo apt install -y libblas-dev libffi-dev libffi7 liblapack-dev libz-dev gfortran
+        # libffi-dev pulls in whichever libffi runtime the image ships; naming
+        # it explicitly breaks whenever the runner image moves (libffi7 does not
+        # exist on ubuntu 24.04, which ubuntu-latest now resolves to).
+        run: sudo apt update && sudo apt install -y libblas-dev libffi-dev liblapack-dev libz-dev gfortran
 
       - name: Build libquil.so
         run: |
           cd $GITHUB_WORKSPACE/libquil
           ls
-          sbcl --noinform --non-interactive --eval '(ql:quickload :sbcl-librarian)'
-          sbcl --dynamic-space-size 8192 --noinform --non-interactive --eval '(ql:quickload :libquil)'
           make
 
       - name: Test
         run: |
-          cd $GITHUB_WORKSPACE/libquil/examples/qvm && LD_LIBRARY_PATH=../.. make test
-          cd $GITHUB_WORKSPACE/libquil/examples/quilc && LD_LIBRARY_PATH=../.. make test
+          cd $GITHUB_WORKSPACE/libquil/examples/qvm && LD_LIBRARY_PATH=../..:../../runtime make test
+          cd $GITHUB_WORKSPACE/libquil/examples/quilc && LD_LIBRARY_PATH=../..:../../runtime make test
 
       - name: Package files
         run: |
-          mkdir -p $GITHUB_WORKSPACE/artifacts/libquil && cp $GITHUB_WORKSPACE/libquil/libquil.h $GITHUB_WORKSPACE/libquil/libquil.so $GITHUB_WORKSPACE/libquil/libquil.core /usr/local/lib/libsbcl.so $GITHUB_WORKSPACE/artifacts/libquil
+          mkdir -p $GITHUB_WORKSPACE/artifacts/libquil
+          cp $GITHUB_WORKSPACE/libquil/libquil.h $GITHUB_WORKSPACE/libquil/libquil.so $GITHUB_WORKSPACE/artifacts/libquil
+          # The runtime directory holds libsbcl_librarian, the core it loads,
+          # libsbcl itself and the runtime headers; they must ship together.
+          cp $GITHUB_WORKSPACE/libquil/runtime/* $GITHUB_WORKSPACE/artifacts/libquil
 
       - name: Store artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: linux-amd64
-          path: ${{ github.workspace }}/artifacts/*
+          # A directory (not a glob): upload-artifact v4 preserves structure
+          # relative to it, keeping the libquil/ prefix the release job unzips.
+          path: ${{ github.workspace }}/artifacts
 
-  build-macos:
-    name: Build libquil.dylib
+  build-macos-arm:
+    name: Build Apple-Silicon libquil.dylib
     runs-on: macos-latest
 
     steps:
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           path: "libquil"
 
       - name: Cache SBCL
         id: cache-sbcl-macos
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: |
             /usr/local/lib/libsbcl.so
             /usr/local/lib/sbcl
             /usr/local/bin/sbcl
             ~/quicklisp
             ~/.sbclrc
-          key: ${{ runner.os }}-build-${{ hashFiles('**/macos-versions' )}}
+          key: ${{ env.ImageOS }}-${{ runner.arch }}-build-${{ hashFiles('**/versions') }}
 
       - uses: Homebrew/actions/setup-homebrew@master
         name: Setup homebrew
@@ -110,37 +151,52 @@ jobs:
         name: Install SBCL, libraries, and quicklisp
         run: |
           brew install sbcl git
-          git clone --branch x86-null-tn https://git.code.sf.net/p/sbcl/sbcl /tmp/sbcl
-          cd /tmp/sbcl && sudo sh make.sh --without-compact-instance-header --without-immobile-space --without-immobile-code && sudo sh make-shared-library.sh
+          # See the Linux job: a linkable runtime has to be built from source. Homebrew's
+          # sbcl is only the cross-compilation host, and its version must be close to the
+          # one being built -- building the old sbcl-2.2.4 with a current host fails in
+          # make-host-1 with "undefined variable: SB-VM::END".
+          git clone --branch sbcl-$SBCL_VERSION https://git.code.sf.net/p/sbcl/sbcl /tmp/sbcl
+          cd /tmp/sbcl && sudo sh make.sh --with-sb-linkable-runtime && sudo sh make-shared-library.sh
           brew remove -f sbcl
           sudo sh install.sh
+          sudo mkdir -p /usr/local/lib
           sudo cp src/runtime/libsbcl.so /usr/local/lib/libsbcl.so
 
           brew install wget
-          wget -P /tmp/ 'https://beta.quicklisp.org/quicklisp.lisp'
-          sbcl --noinform --non-interactive --load /tmp/quicklisp.lisp --eval "(quicklisp-quickstart:install :dist-url \"http://beta.quicklisp.org/dist/quicklisp/2022-04-01/distinfo.txt\")"
-          sbcl --noinform --non-interactive --load ~/quicklisp/setup.lisp --eval '(ql-util:without-prompting (ql:add-to-init-file))'
+          if [ ! -f ~/quicklisp/setup.lisp ]; then
+            wget -P /tmp/ 'https://beta.quicklisp.org/quicklisp.lisp'
+            sbcl --noinform --non-interactive --load /tmp/quicklisp.lisp --eval "(quicklisp-quickstart:install :dist-url \"http://beta.quicklisp.org/dist/quicklisp/$QUICKLISP_VERSION/distinfo.txt\")"
+            sbcl --noinform --non-interactive --load ~/quicklisp/setup.lisp --eval '(ql-util:without-prompting (ql:add-to-init-file))'
+          fi
           echo "#+quicklisp(push (truename \"$GITHUB_WORKSPACE\") ql:*local-project-directories*)" >> ~/.sbclrc
           rm -f /tmp/quicklisp.lisp
           cat ~/.sbclrc
 
       - name: Pull Lisp dependencies
         run: |
-          git clone https://github.com/notmgsk/quilc.git $GITHUB_WORKSPACE/quilc && cd $GITHUB_WORKSPACE/quilc && git checkout bffea7fdb972cc9b3a50b790246b6a3143c88c7a
+          git clone https://github.com/quil-lang/quilc.git $GITHUB_WORKSPACE/quilc && cd $GITHUB_WORKSPACE/quilc && git checkout 3d83b665b3addc33be2f696b1cdebc0c4c5a04f9
           git clone https://github.com/quil-lang/qvm.git $GITHUB_WORKSPACE/qvm && cd $GITHUB_WORKSPACE/qvm && git checkout 4617625cb6053b1adfd3f7aea9cd2be328b225f6
           git clone https://github.com/quil-lang/magicl.git $GITHUB_WORKSPACE/magicl
           git clone https://github.com/stylewarning/cl-permutation $GITHUB_WORKSPACE/cl-permutation
-          git clone https://github.com/quil-lang/sbcl-librarian.git $GITHUB_WORKSPACE/sbcl-librarian && cd $GITHUB_WORKSPACE/sbcl-librarian && git checkout 04f7e390c777084ac43b8df9e90593a041da2381
+          git clone $SBCL_LIBRARIAN_REPO $GITHUB_WORKSPACE/sbcl-librarian && cd $GITHUB_WORKSPACE/sbcl-librarian && git checkout $SBCL_LIBRARIAN_REF
 
       - name: Install quilc dependencies
-        run: brew install lapack openblas libffi gfortran
+        run: |
+          # Deliberately not installing lapack. magicl prefers Homebrew's reference
+          # LAPACK over everything else, and on arm64 it computes incorrect
+          # eigenvectors -- compilation then fails with "Could not find diagonalizer
+          # for matrix ... after 16 attempts". With it absent, magicl falls through to
+          # a bare liblapack.dylib, which these symlinks point at OpenBLAS.
+          brew uninstall --ignore-dependencies lapack || true
+          brew install openblas libffi gfortran
+          sudo mkdir -p /usr/local/lib
+          sudo ln -sf "$(brew --prefix openblas)/lib/libopenblas.dylib" /usr/local/lib/liblapack.dylib
+          sudo ln -sf "$(brew --prefix openblas)/lib/libopenblas.dylib" /usr/local/lib/libblas.dylib
 
       - name: Build libquil.dylib
         run: |
           cd $GITHUB_WORKSPACE/libquil
           ls
-          sbcl --noinform --non-interactive --eval '(ql:quickload :sbcl-librarian)'
-          sbcl --dynamic-space-size 8192 --noinform --non-interactive --eval '(ql:quickload :libquil)'
           make
 
       - name: ls
@@ -149,15 +205,22 @@ jobs:
 
       - name: Test
         run: |
-          cd $GITHUB_WORKSPACE/libquil/examples/qvm && DYLD_LIBRARY_PATH=$GITHUB_WORKSPACE/libquil make test
-          cd $GITHUB_WORKSPACE/libquil/examples/quilc && DYLD_LIBRARY_PATH=$GITHUB_WORKSPACE/libquil make test
+          cd $GITHUB_WORKSPACE/libquil/examples/qvm && DYLD_LIBRARY_PATH=$GITHUB_WORKSPACE/libquil:$GITHUB_WORKSPACE/libquil/runtime make test
+          cd $GITHUB_WORKSPACE/libquil/examples/quilc && DYLD_LIBRARY_PATH=$GITHUB_WORKSPACE/libquil:$GITHUB_WORKSPACE/libquil/runtime make test
 
       - name: Package files
         run: |
-          mkdir -p $GITHUB_WORKSPACE/artifacts/libquil && cp $GITHUB_WORKSPACE/libquil/libquil.h $GITHUB_WORKSPACE/libquil/libquil.dylib $GITHUB_WORKSPACE/libquil/libquil.core /usr/local/lib/libsbcl.so $GITHUB_WORKSPACE/artifacts/libquil
+          mkdir -p $GITHUB_WORKSPACE/artifacts/libquil
+          cp $GITHUB_WORKSPACE/libquil/libquil.h $GITHUB_WORKSPACE/libquil/libquil.dylib $GITHUB_WORKSPACE/artifacts/libquil
+          # The runtime directory holds libsbcl_librarian, the core it loads,
+          # libsbcl itself and the runtime headers; they must ship together.
+          cp $GITHUB_WORKSPACE/libquil/runtime/* $GITHUB_WORKSPACE/artifacts/libquil
 
       - name: Store artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
-          name: macos
-          path: ${{ github.workspace }}/artifacts/*
+          name: macos-arm64
+          # A directory (not a glob): upload-artifact v4 preserves structure
+          # relative to it, keeping the libquil/ prefix the release job unzips.
+          path: ${{ github.workspace }}/artifacts
+

.github/workflows/release.yml

@@ -1,14 +1,9 @@
 name: Release libquil
 
 on:
+  # Dispatching on main cuts a release; dispatching on any other branch cuts a
+  # prerelease, so libquil-sys can be built against a libquil that is not yet stable.
   workflow_dispatch:
-    inputs:
-      type:
-        description: Bump versions and trigger a new release.
-        required: true
-        default: release
-        options:
-          - release
 
 jobs:
   build:
@@ -25,40 +20,61 @@ jobs:
           git config --global user.name "${{ github.triggering_actor }}"
           git config --global user.email "${{ github.triggering_actor }}@users.noreply.github.com"
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          # Release the ref this was dispatched on, not the default branch.
+          ref: ${{ github.ref }}
 
+      # Each artifact is downloaded into its own directory: the two macOS
+      # artifacts contain identically named files, so a shared directory would
+      # have one overwrite the other.
       - name: Download linux artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: linux-amd64
-          path: artifacts/
+          path: artifacts/linux-amd64/
 
       - name: Archive linux artifacts
         run: |
-          cd artifacts && zip linux-amd64.zip libquil/libsbcl.so libquil/libquil.so libquil/libquil.core libquil/libquil.h
+          cd artifacts/linux-amd64 && zip ../linux-amd64.zip libquil/libsbcl.so libquil/libsbcl_librarian.so libquil/libquil.so libquil/libquil.core libquil/libquil.h libquil/sbcl_librarian.h libquil/sbcl_librarian_err.h
 
-      - name: Download macos artifacts
-        uses: actions/download-artifact@v3
+      - name: Download macos arm64 artifacts
+        uses: actions/download-artifact@v4
         with:
-          name: macos
-          path: artifacts/
+          name: macos-arm64
+          path: artifacts/macos-arm64/
 
-      - name: Archive linux artifacts
+      - name: Archive macos arm64 artifacts
         run: |
-          cd artifacts && zip macos.zip libquil/libsbcl.so libquil/libquil.dylib libquil/libquil.core libquil/libquil.h
+          cd artifacts/macos-arm64 && zip ../macos-arm64.zip libquil/libsbcl.so libquil/libsbcl_librarian.dylib libquil/libquil.dylib libquil/libquil.core libquil/libquil.h libquil/sbcl_librarian.h libquil/sbcl_librarian_err.h
 
       - name: List artifacts
         run: |
           ls -R artifacts/
           unzip -l artifacts/linux-amd64.zip
-          unzip -l artifacts/macos.zip
+          unzip -l artifacts/macos-arm64.zip
 
       - name: Install Knope
-        uses: knope-dev/action@v2.0.0
+        uses: knope-dev/action@v2.1.2
         with:
-          version: 0.11.0
+          # 0.23 is needed for `--prerelease-label` and because older versions tag
+          # the default branch rather than the ref being released, which produces a
+          # tag pointing at the wrong code when releasing from a branch.
+          version: 0.23.0 # Test before updating, breaking changes likely: https://github.com/knope-dev/action#install-latest-version
 
       - name: Release
-        run: knope release -v
+        env:
+          # The Release step authenticates with this rather than an argument.
+          GITHUB_TOKEN: ${{ secrets.PAT }}
+        run: |
+          set -euo pipefail
+
+          # A release off main is a real one; anywhere else it can only be a
+          # prerelease, which is how libquil gets published for testing while an ABI
+          # change is still under review.
+          if [[ "$GITHUB_REF" == "refs/heads/main" ]]; then
+            knope release -v
+          else
+            knope release -v --prerelease-label=rc
+          fi

.gitignore

@@ -2,3 +2,7 @@ libquil.c
 libquil.core
 libquil.h
 libquil.so
+libquil.dylib
+libquil.py
+build/
+runtime/