Rework racket based runtime/compilation #7517
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: CI | |
defaults: | |
run: | |
shell: bash | |
on: | |
# Build on every pull request (and new PR commit) | |
pull_request: | |
# Build on new pushes to trunk (E.g. Merge commits) | |
# Without the branch filter, each commit on a branch with a PR is triggered twice. | |
# See: https://github.community/t/how-to-trigger-an-action-on-push-or-pull-request-but-not-both/16662 | |
push: | |
branches: | |
- trunk | |
tags: | |
- release/* | |
workflow_dispatch: | |
env: | |
ormolu_version: "0.5.0.1" | |
racket_version: "8.7" | |
ucm_local_bin: "ucm-local-bin" | |
jit_version: "@unison/internal/releases/0.0.10" | |
jit_src_scheme: "unison-jit-src/scheme-libs/racket" | |
jit_dist: "unison-jit-dist" | |
jit_generator_os: ubuntu-20.04 | |
base-codebase: "~/.cache/unisonlanguage/base.unison" | |
jobs: | |
ormolu: | |
runs-on: ubuntu-20.04 | |
# Only run formatting on trunk commits | |
# This is because the job won't have permission to push back to | |
# contributor forks on contributor PRs. | |
if: github.ref_name == 'trunk' | |
steps: | |
- uses: actions/checkout@v4 | |
- name: Get changed files | |
id: changed-files | |
uses: tj-actions/changed-files@v41 | |
with: | |
# globs copied from default settings for run-ormolu | |
files: | | |
**/*.hs | |
**/*.hs-boot | |
separator: "\n" | |
- uses: haskell-actions/run-ormolu@v14 | |
with: | |
version: ${{ env.ormolu_version }} | |
mode: inplace | |
pattern: ${{ steps.changed-files.outputs.all_changed_files }} | |
- name: apply formatting changes | |
uses: stefanzweifel/git-auto-commit-action@v4 | |
if: ${{ always() }} | |
with: | |
commit_message: automatically run ormolu | |
build-ucm: | |
name: Build UCM ${{ matrix.os }} | |
runs-on: ${{ matrix.os }} | |
if: always() | |
needs: ormolu | |
strategy: | |
# Run each build to completion, regardless of if any have failed | |
fail-fast: false | |
matrix: | |
os: | |
# While iterating on this file, you can disable one or more of these to speed things up | |
- ubuntu-20.04 | |
- macOS-12 | |
- windows-2019 | |
# - windows-2022 | |
steps: | |
- uses: actions/checkout@v4 | |
- name: tweak environment | |
run: | | |
ucm_local_bin="${RUNNER_TEMP//\\//}/${ucm_local_bin}" | |
echo "ucm_local_bin=$ucm_local_bin" >> $GITHUB_ENV | |
if [[ ${{runner.os}} = "Windows" ]]; then | |
echo "ucm=$ucm_local_bin/unison.exe" >> $GITHUB_ENV | |
echo "transcripts=$ucm_local_bin/transcripts.exe" >> $GITHUB_ENV | |
else | |
echo "ucm=$ucm_local_bin/unison" >> $GITHUB_ENV | |
echo "transcripts=$ucm_local_bin/transcripts" >> $GITHUB_ENV | |
fi | |
- name: cache ucm binaries | |
id: cache-ucm-binaries | |
uses: actions/cache@v4 | |
with: | |
path: ${{env.ucm_local_bin}} | |
key: ucm-${{ matrix.os }}-${{ hashFiles('**/stack.yaml', '**/package.yaml', '**/*.hs')}}-${{ hashFiles('**/unison-src/**') }} | |
restore-keys: ucm-${{ matrix.os }}-${{ hashFiles('**/stack.yaml', '**/package.yaml', '**/*.hs') }}- | |
# The number towards the beginning of the cache keys allow you to manually avoid using a previous cache. | |
# GitHub will automatically delete caches that haven't been accessed in 7 days, but there is no way to | |
# purge one manually. | |
- id: stackage-resolver | |
name: record stackage resolver | |
# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#environment-files | |
# looks for `resolver: nightly-yyyy-mm-dd` or `resolver: lts-xx.yy` in `stack.yaml` and splits it into | |
# `nightly` or `lts-xx`. the whole resolver string is put into $resolver as a backup cache key | |
# ${{ env.resolver_short }} | |
# ${{ env.resolver }} | |
run: | | |
grep resolver stack.yaml | awk '{ x="resolver_short="; if (split($2,a,"-") > 2) print x a[1]; else {split($2,b,"."); print x b[1]}}' >> "$GITHUB_ENV" | |
grep resolver stack.yaml | awk '{print "resolver="$2}' >> "$GITHUB_ENV" | |
# Cache ~/.stack, keyed by the contents of 'stack.yaml'. | |
- name: cache ~/.stack (non-Windows) | |
uses: actions/cache@v4 | |
if: runner.os != 'Windows' && steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
with: | |
path: ~/.stack | |
key: stack-1_${{matrix.os}}-${{env.resolver}}-${{hashFiles('**/stack.yaml', '**/package.yaml')}} | |
# Fall-back to use the most recent cache for this resolver | |
restore-keys: stack-1_${{matrix.os}}-${{env.resolver}}- | |
save-always: true | |
# Cache ~/.stack, keyed by the contents of 'stack.yaml'. | |
- name: cache ~/.stack (Windows) | |
uses: actions/cache@v4 | |
if: runner.os == 'Windows' && steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
with: | |
path: | | |
C:\Users\runneradmin\AppData\Roaming\stack | |
C:\Users\runneradmin\AppData\Local\Programs\stack | |
key: stack-1_${{matrix.os}}-${{env.resolver}}-${{hashFiles('**/stack.yaml', '**/package.yaml')}} | |
# Fall-back to use the most recent cache for this resolver | |
restore-keys: stack-1_${{matrix.os}}-${{env.resolver}}- | |
save-always: true | |
# Cache each local package's ~/.stack-work for fast incremental builds in CI. | |
- name: cache .stack-work | |
uses: actions/cache@v4 | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
with: | |
path: | | |
**/.stack-work | |
# Main cache key: commit hash. This should always result in a cache miss... | |
# So when loading a cache we'll always fall back to the restore-keys, | |
# which should load the most recent cache via a prefix search on the most | |
# recent branch cache. | |
# Then it will save a new cache at this commit sha, which should be used by | |
# the next build on this branch. | |
key: stack-work-4_${{matrix.os}}-${{env.resolver}}-${{hashFiles('**/stack.yaml', '**/package.yaml')}}-${{hashFiles('**/*.hs')}} | |
restore-keys: | | |
stack-work-4_${{matrix.os}}-${{env.resolver}}-${{hashFiles('**/stack.yaml', '**/package.yaml')}}- | |
stack-work-4_${{matrix.os}}-${{env.resolver}}- | |
stack-work-4_${{matrix.os}}- | |
save-always: true | |
# Install stack by downloading the binary from GitHub. | |
# The installation process differs by OS. | |
- name: install stack (Linux) | |
if: runner.os == 'Linux' && steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
working-directory: ${{ runner.temp }} | |
run: | | |
mkdir stack && cd stack | |
curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-linux-x86_64.tar.gz | tar -xz | |
echo "$PWD/stack-"* >> $GITHUB_PATH | |
- name: install stack (macOS) | |
if: runner.os == 'macOS' && steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
working-directory: ${{ runner.temp }} | |
run: | | |
mkdir stack && cd stack | |
curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-osx-x86_64.tar.gz | tar -xz | |
echo "$PWD/stack-"* >> $GITHUB_PATH | |
- name: install stack (windows) | |
if: runner.os == 'Windows' && steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
working-directory: ${{ runner.temp }} | |
run: | | |
mkdir stack && cd stack | |
curl -L https://github.com/commercialhaskell/stack/releases/download/v2.9.1/stack-2.9.1-windows-x86_64.tar.gz | tar -xz | |
echo "$PWD/stack-"* >> $GITHUB_PATH | |
# One of the transcripts fails if the user's git name hasn't been set. | |
- name: set git user info | |
run: | | |
git config --global user.name "GitHub Actions" | |
git config --global user.email "actions@github.com" | |
- name: remove ~/.stack/setup-exe-cache on macOS | |
if: runner.os == 'macOS' | |
run: rm -rf ~/.stack/setup-exe-cache | |
# Build deps, then build local code. Splitting it into two steps just allows us to see how much time each step | |
# takes. | |
- name: build dependencies | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
# Run up to 5 times in a row before giving up. | |
# It's very unlikely that our build-dependencies step will fail on most builds, | |
# so if it fails its almost certainly due to a race condition on the Windows | |
# file-system API that stack runs into. Since any successful packages are | |
# cached within a single build, it should get further along on each re-start | |
# and should hopefully finish! | |
run: | | |
stack --version | |
tries=1 | |
if [[ ${{matrix.os}} = "windows-"* ]]; then | |
tries=5 | |
fi | |
for (( i = 0; i < $tries; i++ )); do | |
stack build --fast --only-dependencies && break; | |
done | |
- name: build | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: | | |
stack build \ | |
--fast \ | |
--test \ | |
--no-run-tests \ | |
--local-bin-path ${{env.ucm_local_bin}} \ | |
--copy-bins | |
# Run each test suite (tests and transcripts) | |
- name: unison-cli test | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-cli | |
- name: unison-core tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-core | |
- name: unison-parser-typechecker tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-parser-typechecker | |
- name: unison-sqlite tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-sqlite | |
- name: unison-syntax tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-syntax | |
- name: unison-util-bytes tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-util-bytes | |
- name: unison-util-cache tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-util-cache | |
- name: unison-util-relation tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack build --fast --test unison-util-relation | |
- name: round-trip-tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: | | |
${{env.ucm}} transcript unison-src/transcripts-round-trip/main.md | |
${{env.ucm}} transcript unison-src/transcripts-manual/rewrites.md | |
# Fail if any transcripts cause git diffs. | |
git diff --ignore-cr-at-eol --exit-code \ | |
unison-src/transcripts-round-trip/main.output.md \ | |
unison-src/transcripts-manual/rewrites.output.md | |
- name: transcripts | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: | | |
${{env.transcripts}} | |
# Fail if any transcripts cause git diffs. | |
git diff --ignore-cr-at-eol --exit-code unison-src/transcripts | |
- name: cli-integration-tests | |
if: steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: stack exec cli-integration-tests | |
- name: verify stack ghci startup | |
if: runner.os == 'macOS' && steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: echo | stack ghci | |
- name: cache base codebase | |
id: cache-base-codebase | |
uses: actions/cache@v4 | |
with: | |
path: ${{ env.base-codebase }} | |
# this key probably needs something about the schema version too | |
key: base.unison_${{hashFiles('**/unison-src/builtin-tests/base.md')}}. | |
- name: create base.md codebase | |
if: steps.cache-base-codebase.outputs.cache-hit != 'true' | |
run: ${{env.ucm}} transcript.fork -C ${{env.base-codebase}} -S ${{env.base-codebase}} unison-src/builtin-tests/base.md | |
- name: interpreter tests | |
if: runner.os != 'Windows' && steps.cache-ucm-binaries.outputs.cache-hit != 'true' | |
run: | | |
${{ env.ucm }} transcript.fork -c ${{env.base-codebase}} unison-src/builtin-tests/interpreter-tests.md | |
cat unison-src/builtin-tests/interpreter-tests.output.md | |
git diff --exit-code unison-src/builtin-tests/interpreter-tests.output.md | |
- name: save ucm artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: unison-${{ matrix.os }} | |
path: ${{ env.ucm }} | |
if-no-files-found: error | |
generate-jit-source: | |
if: always() && needs.build-ucm.result == 'success' | |
name: Generate JIT source | |
needs: build-ucm | |
runs-on: ubuntu-20.04 | |
steps: | |
- name: set up environment | |
run: | | |
echo "jit_src_scheme=${{ runner.temp }}/${{ env.jit_src_scheme }}" >> $GITHUB_ENV | |
echo "ucm=${{ runner.temp }}/unison" >> $GITHUB_ENV | |
- uses: actions/cache@v4 | |
name: cache jit source | |
if: runner.os == 'Linux' | |
with: | |
path: ${{ env.jit_src_scheme }} | |
key: jit_src_scheme.racket_${{env.racket_version}}.jit_${{env.jit_version}} | |
- name: check source exists | |
id: jit_src_exists | |
run: | | |
files=(data-info boot-generated simple-wrappers builtin-generated compound-wrappers) | |
all_exist=true | |
for file in "${files[@]}"; do | |
if [[ ! -f "${{ env.jit_src_scheme }}/unison/$file.ss" ]]; then | |
echo "$file does not exist." | |
all_exist=false | |
# Uncomment the next line if you want to stop checking after the first missing file | |
# break | |
fi | |
done | |
if $all_exist; then | |
echo "files_exists=true" >> $GITHUB_OUTPUT | |
else | |
echo "files_exists=false" >> $GITHUB_OUTPUT | |
fi | |
- name: create transcript | |
if: steps.jit_src_exists.outputs.files_exists == 'false' | |
uses: DamianReeves/write-file-action@v1.3 | |
with: | |
path: ${{ runner.temp }}/setup-jit.md | |
write-mode: overwrite | |
contents: | | |
```ucm | |
.> project.create-empty jit-setup | |
jit-setup/main> pull ${{ env.jit_version }} lib.jit | |
``` | |
```unison | |
generateSchemeBoot dir = do | |
saveDataInfoFile dir dataInfos | |
saveBaseFile dir bootSpec | |
saveWrapperFile dir simpleWrapperSpec | |
saveBaseFile dir builtinSpec | |
saveWrapperFile dir compoundWrapperSpec | |
go = generateSchemeBoot "${{ env.jit_src_scheme }}" | |
``` | |
```ucm | |
jit-setup/main> run go | |
``` | |
- name: download ucm artifact | |
if: steps.jit_src_exists.outputs.files_exists == 'false' | |
uses: actions/download-artifact@v4 | |
with: | |
name: unison-${{ env.jit_generator_os }} | |
path: ${{ runner.temp }} | |
- name: set ucm permissions | |
if: steps.jit_src_exists.outputs.files_exists == 'false' | |
run: chmod +x ${{ env.ucm }} | |
- name: download scheme-libs | |
if: steps.jit_src_exists.outputs.files_exists == 'false' | |
uses: actions/checkout@v4 | |
- name: generate source | |
if: steps.jit_src_exists.outputs.files_exists == 'false' | |
run: | | |
mkdir -p ${{ env.jit_src_scheme }} | |
cp -R scheme-libs/racket/* ${{ env.jit_src_scheme }} | |
${{ env.ucm }} transcript ${{ runner.temp }}/setup-jit.md | |
- name: save jit source | |
if: ${{ always() }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: jit-source | |
path: ${{ env.jit_src_scheme }}/** | |
if-no-files-found: error | |
build-jit-binary: | |
if: always() && needs.generate-jit-source.result == 'success' | |
name: Build JIT binary ${{ matrix.os }} | |
needs: generate-jit-source | |
runs-on: ${{ matrix.os }} | |
strategy: | |
# Run each build to completion, regardless of if any have failed | |
fail-fast: false | |
matrix: | |
os: | |
# While iterating on this file, you can disable one or more of these to speed things up | |
- ubuntu-20.04 | |
- macOS-12 | |
- windows-2019 | |
steps: | |
- name: set up environment | |
id: checks | |
run: | | |
jit_src_scheme="${{ runner.temp }}/${{ env.jit_src_scheme }}" # scheme source | |
jit_exe="${jit_src_scheme}/unison-runtime" # initially built jit | |
jit_dist="${{ runner.temp }}/${{ env.jit_dist }}" # jit binary with libraries destination | |
jit_dist_exe="${jit_dist}/bin/unison-runtime" # jit binary itself | |
ucm="${{ runner.temp }}/unison" | |
if [[ ${{runner.os}} = "Windows" ]]; then | |
jit_src_scheme="${jit_src_scheme//\\//}" | |
jit_dist="${jit_dist//\\//}" | |
jit_exe="${jit_exe//\\//}.exe" | |
jit_dist_exe="${jit_dist//\\//}/unison-runtime.exe" | |
ucm="${ucm//\\//}.exe" | |
fi | |
echo "jit_src_scheme=$jit_src_scheme" >> $GITHUB_ENV | |
echo "jit_exe=$jit_exe" >> $GITHUB_ENV | |
echo "jit_dist=$jit_dist" >> $GITHUB_ENV | |
echo "jit_dist_exe=$jit_dist_exe" >> $GITHUB_ENV | |
echo "ucm=$ucm" >> $GITHUB_ENV | |
- name: restore jit binaries | |
id: restore-jit-binaries | |
uses: actions/cache@v4 | |
with: | |
path: ${{ env.jit_dist }} | |
key: jit_dist.racket_${{ env.racket_version }}.jit_${{ env.jit_version }} | |
- name: Cache Racket dependencies | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.cache/racket | |
~/.local/share/racket | |
key: ${{ runner.os }}-racket-${{env.racket_version}} | |
- uses: Bogdanp/setup-racket@v1.11 | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
with: | |
architecture: 'x64' | |
distribution: 'full' | |
variant: 'CS' | |
version: ${{env.racket_version}} | |
- uses: awalsh128/cache-apt-pkgs-action@latest | |
if: runner.os == 'Linux' && steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
# read this if a package isn't installing correctly | |
# https://github.com/awalsh128/cache-apt-pkgs-action#caveats | |
with: | |
packages: libb2-dev | |
version: 1.0 # cache key version afaik | |
- name: download jit source | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
uses: actions/download-artifact@v4 | |
with: | |
name: jit-source | |
path: ${{ env.jit_src_scheme }} | |
- uses: actions/checkout@v4 # checkout scheme-libs from unison repo | |
- name: build jit binary | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
shell: bash | |
run: | | |
cp -R scheme-libs/racket/* "$jit_src_scheme" | |
raco pkg install --auto --skip-installed "$jit_src_scheme"/unison | |
raco exe --embed-dlls "$jit_src_scheme"/unison-runtime.rkt | |
raco distribute "$jit_dist" "$jit_exe" | |
- name: save jit binary | |
uses: actions/upload-artifact@v4 | |
with: | |
name: jit-binary-${{ matrix.os }} | |
path: ${{ env.jit_dist }}/** | |
- name: download ucm | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
uses: actions/download-artifact@v4 | |
with: | |
name: unison-${{ matrix.os }} | |
path: ${{ runner.temp }} | |
- name: set ucm permissions | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
run: chmod +x ${{ env.ucm }} | |
- name: get base codebase | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
uses: actions/cache/restore@v4 | |
with: | |
path: ${{ env.base-codebase}} | |
key: base.unison_${{hashFiles('**/unison-src/builtin-tests/base.md')}}. | |
- name: jit integration test ${{ matrix.os }} | |
if: steps.restore-jit-binaries.outputs.cache-hit != 'true' | |
run: | | |
${{ env.ucm }} transcript.fork --runtime-path ${{ env.jit_dist_exe }} -c ${{env.base-codebase}} unison-src/builtin-tests/jit-tests.md | |
cat unison-src/builtin-tests/jit-tests.output.md | |
git diff --exit-code unison-src/builtin-tests/jit-tests.output.md | |
- name: Setup tmate session | |
uses: mxschmitt/action-tmate@v3 | |
if: ${{ failure() }} | |
# timeout-minutes: 15 |