ci.yml

---
name: CI

on: [ push, pull_request ]

env:
  pcre2: pcre2-10.40
  wc: wc
  build: build
  cvtpcre: build/test/retest
  prefix: prefix
  maintainer: kate@elide.org

jobs:
  checkout:
    name: "Checkout"
    runs-on: ubuntu-latest

    steps:
    - name: Cache checkout
      uses: actions/cache@v3
      id: cache-checkout
      with:
        path: ${{ env.wc }}
        key: checkout-${{ github.sha }}

    - name: Checkout
      if: steps.cache-checkout.outputs.cache-hit != 'true'
      uses: actions/checkout@v3
      with:
        submodules: recursive
        path: ${{ env.wc }}

    - name: Uncouple docs test
      if: steps.cache-checkout.outputs.cache-hit != 'true'
      run: sed -ie 's,^test::,doc-test::,' ${{ env.wc }}/share/mk/man.mk

    - name: chmod read only
      if: steps.cache-checkout.outputs.cache-hit != 'true'
      run: |
        chmod -R ug-w ${{ env.wc }}
        chmod -R ug+w ${{ env.wc }}/.git # post-checkout sets git config

  pcre_suite:
    name: "Import PCRE suite"
    runs-on: ubuntu-latest
    needs: [ build ] # for cvtpcre

    steps:
    - name: Cache PCRE suite
      uses: actions/cache@v3
      id: cache-pcre
      with:
        path: pcre-suite/${{ env.pcre2 }}
        key: ${{ env.pcre2 }}

    - name: Fetch PCRE suite
      if: steps.cache-pcre.outputs.cache-hit != 'true'
      run: |
        wget -nv -P pcre-suite/ https://github.com/PCRE2Project/pcre2/releases/download/${{ env.pcre2 }}/${{ env.pcre2 }}.zip
        unzip -d pcre-suite/ pcre-suite/${{ env.pcre2 }}.zip "${{ env.pcre2 }}/testdata/*"

    - name: chmod read only
      if: steps.cache-pcre.outputs.cache-hit != 'true'
      run: |
        chmod -R ug-w pcre-suite

    - name: Cache converted PCRE tests
      uses: actions/cache@v3
      id: cache-cvtpcre
      with:
        path: ${{ env.cvtpcre }}
        key: cvtpcre-bmake-ubuntu-gcc-DEBUG-ASAN-${{ github.sha }}-${{ env.pcre2 }}

    - name: Fetch build
      if: steps.cache-cvtpcre.outputs.cache-hit != 'true'
      uses: actions/cache@v3
      id: cache-build
      with:
        path: ${{ env.build }}
        key: build-bmake-ubuntu-gcc-DEBUG-ASAN-${{ github.sha }} # arbitary build, just for cvtpcre

    - name: Convert PCRE suite
      if: steps.cache-cvtpcre.outputs.cache-hit != 'true'
      run: |
        mkdir -p ${{ env.cvtpcre }}
        # the regexps skipped here take too long to compile in CI at the moment
        for i in $(cd pcre-suite/${{ env.pcre2 }}/testdata; ls -1 testinput*); do
            ./${{ env.build }}/bin/cvtpcre \
                -s 'word (?:[a-zA-Z0-9]+ ){0,300}otherword' \
                -s 'Z*(|d*){216}' \
                -s 'X?(R||){3335}' \
                -s '(|]+){2,2452}' \
                -s 'z{65536}' \
            < pcre-suite/${{ env.pcre2 }}/testdata/$i \
            > ./${{ env.cvtpcre }}/$i.tst
        done
        # things which will probably never be relevant for us:
        rm ${{ env.cvtpcre }}/testinput2.tst # pcre options
        rm ${{ env.cvtpcre }}/testinput3.tst # locale
        rm ${{ env.cvtpcre }}/testinput5.tst # pcre options
        rm ${{ env.cvtpcre }}/testinput8.tst # pcre bytecode
        rm ${{ env.cvtpcre }}/testinput9.tst # character encodings
        rm ${{ env.cvtpcre }}/testinput10.tst # character encodings
        rm ${{ env.cvtpcre }}/testinput11.tst # character encodings
        rm ${{ env.cvtpcre }}/testinput12.tst # character encodings
        rm ${{ env.cvtpcre }}/testinput13.tst # character encodings
        rm ${{ env.cvtpcre }}/testinput15.tst # backtracking limits
        rm ${{ env.cvtpcre }}/testinput16.tst # JIT options
        rm ${{ env.cvtpcre }}/testinput17.tst # JIT options
        rm ${{ env.cvtpcre }}/testinput20.tst # serialisation
        rm ${{ env.cvtpcre }}/testinput21.tst # \C
        rm ${{ env.cvtpcre }}/testinput22.tst # \C
        rm ${{ env.cvtpcre }}/testinput23.tst # \C
        rm ${{ env.cvtpcre }}/testinputEBC.tst # EBCDIC
        # things we should eventually support, but don't yet:
        rm ${{ env.cvtpcre }}/testinput4.tst # we don't provide UTF8 yet
        rm ${{ env.cvtpcre }}/testinput7.tst # we don't provide UTF8 yet
        rm ${{ env.cvtpcre }}/testinput14.tst # we don't provide UTF8 yet
        rm ${{ env.cvtpcre }}/testinput18.tst # POSIX dialect
        rm ${{ env.cvtpcre }}/testinput19.tst # POSIX dialect
        rm ${{ env.cvtpcre }}/testinput24.tst # POSIX dialect
        rm ${{ env.cvtpcre }}/testinput25.tst # POSIX dialect

    - name: chmod read only
      if: steps.cache-cvtpcre.outputs.cache-hit != 'true'
      run: |
        chmod -R ug-w ${{ env.cvtpcre }}

  build:
    name: "Build ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}"
    runs-on: ${{ matrix.os }}-latest
    needs: [ checkout ]

    strategy:
      fail-fast: true
      matrix:
        san: [ NO_SANITIZER, ASAN, UBSAN, MSAN, EFENCE ] # NO_SANITIZER=1 is a no-op
        os: [ ubuntu ]
        cc: [ clang, gcc ]
        make: [ bmake ] # we test makefiles separately
        debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op
        exclude:
          - os: macos
            cc: gcc # it's clang anyway
          - os: macos
            san: EFENCE # not packaged
          - os: macos
            san: MSAN # not supported
          - os: macos
            make: pmake # not packaged

    steps:
    - name: Fetch checkout
      uses: actions/cache@v3
      id: cache-checkout
      with:
        path: ${{ env.wc }}
        key: checkout-${{ github.sha }}

    - name: Cache build
      uses: actions/cache@v3
      id: cache-build
      with:
        path: ${{ env.build }}
        key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }}

    - name: Dependencies (Ubuntu)
      if: matrix.os == 'ubuntu' && steps.cache-build.outputs.cache-hit != 'true'
      run: |
        uname -a
        sudo apt-get install bmake electric-fence
        ${{ matrix.cc }} --version

    - name: Dependencies (MacOS)
      if: matrix.os == 'macos' && steps.cache-build.outputs.cache-hit != 'true'
      run: |
        uname -a
        brew update
        brew install bmake pcre
        ${{ matrix.cc }} --version

    - name: Get number of CPU cores
      if: steps.cache-build.outputs.cache-hit != 'true'
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    - name: Make
      if: steps.cache-build.outputs.cache-hit != 'true'
      run: |
        # note: lexer.h first, because parser.? depends on it
        find . -name 'lexer.?' -exec touch '{}' \; # workaround for git checkout timestamps
        find . -name 'parser.?' -exec touch '{}' \; # workaround for git checkout timestamps
        ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 PKGCONF=pkg-config CC=${{ matrix.cc }} NODOC=1

  # testing different bmake dialects
  # the goal here is to excercise the build system, not the code
  # we don't care about e.g. different compilers here
  test_makefiles:
    name: "Test (Makefiles) ${{ matrix.make }} ${{ matrix.os }} ${{ matrix.debug }}"
    runs-on: ${{ matrix.os }}-latest
    needs: [ checkout ]

    strategy:
      fail-fast: false
      matrix:
        os: [ ubuntu ]
        cc: [ clang ]
        make: [ bmake, pmake ]
        debug: [ DEBUG, RELEASE ] # RELEASE=1 is a no-op
        exclude:
          - os: macos
            make: pmake # not packaged

    steps:
    - name: Fetch checkout
      uses: actions/cache@v3
      id: cache-checkout
      with:
        path: ${{ env.wc }}
        key: checkout-${{ github.sha }}

    - name: Dependencies (Ubuntu)
      if: matrix.os == 'ubuntu'
      run: |
        uname -a
        sudo apt-get install pmake bmake pcregrep
        ${{ matrix.cc }} --version

    - name: Dependencies (MacOS)
      if: matrix.os == 'macos'
      run: |
        uname -a
        brew update
        brew install bmake pcre
        ${{ matrix.cc }} --version

    - name: Get number of CPU cores
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    - name: Make
      run: |
        # note: lexer.h first, because parser.? depends on it
        find . -name 'lexer.?' -exec touch '{}' \; # workaround for git checkout timestamps
        find . -name 'parser.?' -exec touch '{}' \; # workaround for git checkout timestamps
        ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.debug }}=1 PKGCONF=pkg-config CC=${{ matrix.cc }}

    - name: Test
      # I don't want to build SID just for sake of its -l test
      # Same for lx
      run: ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.debug }}=1 PKGCONF=pkg-config SID='true; echo sid' LX='true; echo lx' CC=${{ matrix.cc }} NODOC=1 test

    - name: Install
      run: ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.debug }}=1 PKGCONF=pkg-config PREFIX=../${{ env.prefix }} NODOC=1 install

  test_san:
    name: "Test (Sanitizers) ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}"
    runs-on: ${{ matrix.os }}-latest
    needs: [ build ]

    strategy:
      fail-fast: false
      matrix:
        san: [ ASAN, UBSAN, MSAN, EFENCE ]
        os: [ ubuntu ]
        cc: [ clang, gcc ]
        make: [ bmake ]
        debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op
        exclude:
          - os: macos
            cc: gcc # it's clang anyway
          - os: macos
            san: EFENCE # not packaged
          - os: macos
            san: MSAN # not supported

    steps:
    - name: Fetch checkout
      uses: actions/cache@v3
      id: cache-checkout
      with:
        path: ${{ env.wc }}
        key: checkout-${{ github.sha }}

    - name: Dependencies (Ubuntu)
      if: matrix.os == 'ubuntu'
      run: |
        uname -a
        sudo apt-get install bmake pcregrep electric-fence
        ${{ matrix.cc }} --version

    - name: Dependencies (MacOS)
      if: matrix.os == 'macos'
      run: |
        uname -a
        brew update
        brew install bmake pcre
        ${{ matrix.cc }} --version

    - name: Fetch build
      uses: actions/cache@v3
      id: cache-build
      with:
        path: ${{ env.build }}
        key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }}

    - name: Get number of CPU cores
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    - name: Test
      env:
        EF_DISABLE_BANNER: 1
      # I don't want to build SID just for sake of its -l test
      run: ${{ matrix.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} ${{ matrix.san }}=1 ${{ matrix.debug }}=1 PKGCONF=pkg-config SID='true; echo sid' CC=${{ matrix.cc }} NODOC=1 LX=../${{ env.build }}/bin/lx test

  test_pcre:
    name: "Test (PCRE suite) ${{ matrix.lang }} ${{ matrix.san }} ${{ matrix.cc }} ${{ matrix.os }} ${{ matrix.debug }}"
    runs-on: ${{ matrix.os }}-latest
    needs: [ pcre_suite ] # and also build, but pcre_suite gives us that

    strategy:
      fail-fast: false
      matrix:
        san: [ ASAN, UBSAN, MSAN, EFENCE ]
        os: [ ubuntu ]
        cc: [ clang, gcc ]
        make: [ bmake ]
        debug: [ DEBUG, EXPENSIVE_CHECKS, RELEASE ] # RELEASE=1 is a no-op
        lang: [ "vm -x v1", "vm -x v2", asm, c, vmc, vmops ]
        exclude:
          - os: macos
            cc: gcc # it's clang anyway
          - os: macos
            san: EFENCE # not packaged
          - os: macos
            san: MSAN # not supported

    steps:
    - name: Dependencies (Ubuntu)
      if: matrix.os == 'ubuntu' && matrix.san == 'EFENCE'
      run: |
        uname -a
        sudo apt-get install electric-fence
        ${{ matrix.cc }} --version

    - name: Dependencies (MacOS)
      if: matrix.os == 'macos' && false
      run: |
        uname -a
        brew update
        ${{ matrix.cc }} --version

    - name: Fetch build
      uses: actions/cache@v3
      id: cache-build
      with:
        path: ${{ env.build }}
        key: build-${{ matrix.make }}-${{ matrix.os }}-${{ matrix.cc }}-${{ matrix.debug }}-${{ matrix.san }}-${{ github.sha }}

    - name: Fetch converted PCRE tests
      uses: actions/cache@v3
      id: cache-cvtpcre
      with:
        path: ${{ env.cvtpcre }}
        key: cvtpcre-bmake-ubuntu-gcc-DEBUG-ASAN-${{ github.sha }}-${{ env.pcre2 }}

    - name: Run PCRE suite (${{ matrix.lang }})
      run: CC=${{ matrix.cc }} ./${{ env.build }}/bin/retest -O1 -l ${{ matrix.lang }} ${{ env.cvtpcre }}/*.tst

  docs:
    name: Documentation
    runs-on: ubuntu-latest
    needs: [ checkout ]

    env:
      make: bmake

    steps:
    - name: Cache docs
      uses: actions/cache@v3
      id: cache-docs
      with:
        path: ${{ env.build }}
        key: docs-${{ github.sha }}

    - name: Dependencies (Ubuntu)
      if: steps.cache-docs.outputs.cache-hit != 'true'
      run: |
        uname -a
        sudo apt-get update
        sudo apt-get install bmake libxml2-utils xsltproc docbook-xml docbook-xsl

    - name: Fetch checkout
      if: steps.cache-docs.outputs.cache-hit != 'true'
      uses: actions/cache@v3
      id: cache-checkout
      with:
        path: ${{ env.wc }}
        key: checkout-${{ github.sha }}

    - name: Get number of CPU cores
      if: steps.cache-docs.outputs.cache-hit != 'true'
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    - name: Test docs
      if: steps.cache-docs.outputs.cache-hit != 'true'
      run: ${{ env.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} doc-test

    - name: make doc
      if: steps.cache-docs.outputs.cache-hit != 'true'
      run: |
        mkdir -p ${{ env.build }}/man # XXX
        ${{ env.make }} -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} doc

    - name: chmod read only
      if: steps.cache-docs.outputs.cache-hit != 'true'
      run: |
        chmod -R ug-w ${{ env.build }}

  install:
    name: Install
    runs-on: ubuntu-latest
    needs: [ build, docs ]

    env:
      san: NO_SANITIZER # NO_SANITIZER=1 is a no-op
      os: ubuntu
      cc: clang
      make: bmake
      debug: RELEASE # RELEASE=1 is a no-op

    steps:
    - name: Cache prefix
      uses: actions/cache@v3
      id: cache-prefix
      with:
        path: ${{ env.prefix }}
        key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }}

    - name: Dependencies (Ubuntu)
      if: env.os == 'ubuntu' && steps.cache-prefix.outputs.cache-hit != 'true'
      run: |
        uname -a
        sudo apt-get install bmake

    - name: Fetch checkout
      if: steps.cache-prefix.outputs.cache-hit != 'true'
      uses: actions/cache@v3
      id: cache-checkout
      with:
        path: ${{ env.wc }}
        key: checkout-${{ github.sha }}

    - name: Fetch build
      if: steps.cache-prefix.outputs.cache-hit != 'true'
      uses: actions/cache@v3
      id: cache-build
      with:
        path: ${{ env.build }}
        key: build-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }}

    - name: Fetch docs
      if: steps.cache-prefix.outputs.cache-hit != 'true'
      uses: actions/cache@v3
      id: cache-docs
      with:
        path: ${{ env.build }}
        key: docs-${{ github.sha }}

    - name: Get number of CPU cores
      if: steps.cache-prefix.outputs.cache-hit != 'true'
      uses: SimenB/github-actions-cpu-cores@v1
      id: cpu-cores

    - name: Install
      if: steps.cache-prefix.outputs.cache-hit != 'true'
      run: |
        chmod ug+w ${{ env.build }}/pc # XXX: written to during make install
        bmake -r -j $((${{ steps.cpu-cores.outputs.count }} + 1)) -C ${{ env.wc }} BUILD=../${{ env.build }} PREFIX=../${{ env.prefix }}/usr install

  fpm:
    name: Package ${{ matrix.pkg }}
    runs-on: ubuntu-latest
    needs: [ install ]

    env:
      san: NO_SANITIZER # NO_SANITIZER=1 is a no-op
      os: ubuntu
      cc: clang
      make: bmake
      debug: RELEASE # RELEASE=1 is a no-op

    strategy:
      fail-fast: false
      matrix:
        # others: cpan, gem, npm, osxpkg, p5p, pear, pkgin, pleaserun, puppet, python, snap, solaris, virtualenv ]
        pkg: [ apk, deb, freebsd, rpm, sh, tar, zip ] # XXX: pacman is broken currently

    steps:
    - name: Dependencies (Ubuntu)
      if: env.os == 'ubuntu'
      run: |
        uname -a
        sudo gem install --no-document fpm
        fpm -v

    - name: Fetch prefix
      uses: actions/cache@v3
      id: cache-prefix
      with:
        path: ${{ env.prefix }}
        key: prefix-${{ env.make }}-${{ env.os }}-${{ env.cc }}-${{ env.debug }}-${{ env.san }}-${{ github.sha }}

    - name: Find version
      # TODO: would get a tag or branch name here
      run: |
        printf "package_version=%s\n" 0.$(echo ${{ github.sha }} | cut -c-7) >> $GITHUB_ENV
        printf "package_name=%s\n" $(echo ${{ github.repository }} | cut -f2 -d/) >> $GITHUB_ENV

    - name: Package
      run: |
        mkdir pkg
        fpm --log warn -C ${{ env.prefix }} -p pkg/ -n ${{ env.package_name }} -t ${{ matrix.pkg }} -v ${{ env.package_version }} -a $(uname -m) -m ${{ env.maintainer }} -s dir \
        | ruby -e 'h = eval(STDIN.read); puts h[:level].to_s.sub("fatal", "::error::").sub("warn", "::warning::") + " " + h[:message]'
        # now there's a single file in pkg/ (because this is a matrix build), name format differs per package type
        [ -f pkg/*.sh -o -f pkg/*.tar ] && gzip -9 pkg/*
        [ -f pkg/${{ env.package_name }}.* ] && mv pkg/* pkg/$(basename pkg/* | sed 's,^${{ env.package_name }}\.,${{ env.package_name }}-${{ env.package_version }}.,') # fix up naming for xyz.{sh,tar,zip}
        printf "package_file=%s\n" $(basename pkg/*) >> $GITHUB_ENV

    - name: Upload
      uses: actions/upload-artifact@v3
      with:
        name: ${{ env.package_file }}
        path: pkg/${{ env.package_file }}