pipeline.yml

steps:
  # first, test supported Julia versions (using local CUDA as installed on the system)
  - group: ":julia: Julia"
    key: "julia"
    steps:
      - label: "Julia {{matrix.julia}}"
        plugins:
          - JuliaCI/julia#v1:
              version: "{{matrix.julia}}"
          - JuliaCI/julia-test#v1:
              test_args: "--quickfail"
          - JuliaCI/julia-coverage#v1:
              dirs:
                - src
                - lib
                - examples
        agents:
          queue: "juliagpu"
          cuda: "*"
        commands: |
          julia --project -e '
              # make sure the 1.7-era Manifest works on this Julia version
              using Pkg
              Pkg.resolve()

              # use the CUDA installation from the CI image
              using CUDA
              CUDA.set_runtime_version!("local")'
        if: build.message !~ /\[skip tests\]/ &&
            build.message !~ /\[skip julia\]/
        timeout_in_minutes: 120
        matrix:
          setup:
            julia:
              - "1.7"
              - "1.8"
              - "1.9"
              - "1.10-nightly"
              - "nightly"
          adjustments:
            - with:
                julia: "1.10-nightly"
              soft_fail: true
            - with:
                julia: "nightly"
              soft_fail: true

  # then, test supported CUDA toolkits (installed through the artifact system)
  - group: "CUDA"
    key: "cuda"
    depends_on: "julia"
    steps:
      - label: "CUDA {{matrix.cuda}}"
        plugins:
          - JuliaCI/julia#v1:
              version: 1.9
          - JuliaCI/julia-test#v1:
              test_args: "core base libraries"
          - JuliaCI/julia-coverage#v1:
              dirs:
                - src
                - lib
                - examples
        agents:
          queue: "juliagpu"
          cuda: "*"
        if: build.message !~ /\[skip tests\]/ &&
            build.message !~ /\[skip cuda\]/ &&
            !build.pull_request.draft
        timeout_in_minutes: 120
        matrix:
          setup:
            cuda:
              - "12.2"
              - "12.1"
              - "12.0"
              - "11.8"
              - "11.7"
              - "11.6"
              - "11.5"
              - "11.4"
              - "11.3"
              - "11.2"
              - "11.1"
              - "11.0"
        commands: |
          julia --project -e 'using CUDA; CUDA.set_runtime_version!(v"{{matrix.cuda}}")'

  - group: ":nesting_dolls: Subpackages"
    depends_on: "cuda"
    steps:
      - label: "{{matrix.package}} on CUDA {{matrix.cuda}}"
        matrix:
          setup:
            cuda:
              - "11.0"
            package:
              - "cuDNN"
              - "cuTENSOR"
              - "cuStateVec"
              - "cuTensorNet"
          adjustments:
            - with:
                cuda: "12.0"
                package: "cuDNN"
            - with:
                cuda: "12.0"
                package: "cuTENSOR"
        plugins:
          - JuliaCI/julia#v1:
              version: "1.9"
          - JuliaCI/julia-coverage#v1:
              dirs:
                - src
                - lib
                - examples
        agents:
          queue: "juliagpu"
          cuda: "*"
        if: build.message !~ /\[skip tests\]/ &&
            build.message !~ /\[skip subpackages\]/ &&
            !build.pull_request.draft
        timeout_in_minutes: 120
        commands: |
          julia -e '
            using Pkg

            println("--- :julia: Instantiating project")
            Pkg.activate(joinpath(pwd(), "lib", lowercase("{{matrix.package}}")))
            if "{{matrix.package}}" == "cuTensorNet"
              # HACK: cuTensorNet depends on a development version of cuTENSOR
              Pkg.develop(path=joinpath(pwd(), "lib", "cutensor"))
            end
            Pkg.develop(path=pwd())
            Pkg.instantiate()

            using CUDA
            CUDA.set_runtime_version!(v"{{matrix.cuda}}")

            println("+++ :julia: Running tests")
            Pkg.test()'

  - group: ":telescope: Downstream"
    depends_on: "cuda"
    steps:
      - label: "NNlibCUDA.jl on CUDA 12"
        plugins:
          - JuliaCI/julia#v1:
              version: 1.9
          - JuliaCI/julia-coverage#v1:
              dirs:
                - src
                - lib
                - examples
        command: |
          julia -e '
            using Pkg

            println("--- :julia: Instantiating project")
            Pkg.develop(path=pwd())
            Pkg.develop(path=joinpath(pwd(), "lib", "cudnn"))
            Pkg.add(name="NNlibCUDA", rev="master")
            Pkg.instantiate()

            using CUDA
            CUDA.set_runtime_version!(v"12.0")

            println("+++ :julia: Running tests")
            Pkg.test("NNlibCUDA"; coverage=true)'
        agents:
          queue: "juliagpu"
          cuda: "*"
        if: build.message !~ /\[skip tests\]/ &&
            build.message !~ /\[skip downstream\]/ &&
            !build.pull_request.draft
        timeout_in_minutes: 60

  - group: ":eyes: Special"
    depends_on: "cuda"
    steps:
      - label: "GPU-less environment"
        plugins:
          - JuliaCI/julia#v1:
              version: 1.9
          - JuliaCI/julia-coverage#v1:
              dirs:
                - src
                - lib
                - examples
          - JuliaCI/julia-test#v1:
              run_tests: false
        command: |
          julia --project -e '
            using CUDA
            @assert !CUDA.functional()
            @assert !isdefined(CUDA, :libcudart)
            CUDA.set_runtime_version!(v"11.6")'
          julia --project -e '
            using CUDA
            @assert !CUDA.functional()
            @assert isdefined(CUDA, :libcudart)'
        agents:
          queue: "juliagpu"
          intel: "*"
        if: build.message !~ /\[skip tests\]/ && !build.pull_request.draft
        timeout_in_minutes: 60

      - label: "Documentation"
        plugins:
          - JuliaCI/julia#v1:
              version: 1.9
        command: |
          julia --project -e '
            println("--- :julia: Instantiating project")
            using Pkg
            Pkg.instantiate()
            Pkg.activate("docs")
            Pkg.instantiate()
            push!(LOAD_PATH, @__DIR__)

            println("+++ :julia: Building documentation")
            include("docs/make.jl")'
        agents:
          queue: "juliagpu"
          cuda: "*"
        if: build.message !~ /\[skip docs\]/ &&
            !build.pull_request.draft
        timeout_in_minutes: 30

      - label: "Compute sanitizer"
        plugins:
          - JuliaCI/julia#v1:
              version: 1.9
          - JuliaCI/julia-test#v1:
              test_args: "--sanitize core"
          - JuliaCI/julia-coverage#v1:
              dirs:
                - src
                - lib
                - examples
        agents:
          queue: "juliagpu"
          cuda: "*"
        env:
          JULIA_CUDA_USE_COMPAT: 'false'  # NVIDIA bug #3418723: injection tools prevent probing libcuda
        if: build.message !~ /\[skip tests\]/ &&
            build.message !~ /\[skip sanitizer\]/ &&
            !build.pull_request.draft
        timeout_in_minutes: 10

  # we want to benchmark every commit on the master branch, even if it failed CI
  - wait: ~
    continue_on_failure: true

  - group: ":racehorse: Benchmarks"
    steps:
      # benchmarks outside of the master branch don't submit their results,
      # so they can run on any system in the juliagpu queue.
      - label: "Benchmarks (dry run)"
        plugins:
          - JuliaCI/julia#v1:
              version: 1.9
        command: |
          julia --project -e '
            println("--- :julia: Instantiating project")
            using Pkg
            Pkg.instantiate()
            Pkg.activate("perf")
            Pkg.instantiate()
            push!(LOAD_PATH, @__DIR__)

            println("+++ :julia: Benchmarking")
            include("perf/runbenchmarks.jl")'
        agents:
          queue: "juliagpu"
          cuda: "*"
        if: build.message !~ /\[skip benchmarks\]/ &&
            build.branch !~ /^master$$/ &&
            !build.pull_request.draft
        timeout_in_minutes: 30

      # if we will submit results, use the benchmark queue so that we will
      # be running on the same system each time
      - label: "Benchmarks on Julia {{matrix.julia}}"
        plugins:
          - JuliaCI/julia#v1:
              version: "{{matrix.julia}}"
        env:
          BENCHMARKS: "true"
          CODESPEED_PROJECT: "$BUILDKITE_PIPELINE_NAME"
          CODESPEED_BRANCH: "$BUILDKITE_BRANCH"
          CODESPEED_COMMIT: "$BUILDKITE_COMMIT"
          CODESPEED_EXECUTABLE: "Julia {{matrix.julia}}"
        command: |
          julia --project -e '
            ENV["CODESPEED_ENVIRONMENT"] = ENV["BUILDKITE_AGENT_NAME"]

            println("--- :julia: Instantiating project")
            using Pkg
            Pkg.resolve()
            Pkg.instantiate()
            Pkg.activate("perf")
            Pkg.resolve()
            Pkg.instantiate()
            push!(LOAD_PATH, @__DIR__)

            println("+++ :julia: Benchmarking")
            include("perf/runbenchmarks.jl")'
        agents:
          queue: "benchmark"
          cuda: "*"
        if: build.message !~ /\[skip benchmarks\]/ &&
            build.branch =~ /^master$$/
        matrix:
          setup:
            julia:
              - "1.7"
              - "1.8"
              - "1.9"
              - "1.10"
              - "nightly"
          adjustments:
            - with:
                julia: "nightly"
              soft_fail: true
        timeout_in_minutes: 30

env:
  JULIA_PKG_SERVER: "" # we don't want to wait until the PkgServer updates
  SECRET_CODECOV_TOKEN: "o43LveCldXhMUteqc2AzdL+dx51CUYz2L/6j+WjSr4lbLb0t/SfpWEPJzrymGgYnQ09hR7otPNShA+40ew6yRMRV7TSgm6ZgdbNeum7drjEMAqX3knKgqbTtvPqgjkQiaXNzSWuJfrsfFb32bkj/iXAoIEdBrpVSYHCb95cf1O10LBTFRlH8AAGzsk9Tvc1cIOl3Wdp34KDLKwIzFYTF405QhgtBwwopaC5DRIuUNuLJ/wPOif4rF1OOMUkPCiTFb2HflpGRhiRvKuj34WwNu/+HEE2pmmLQnKPcJOsg+iPvFY4U/prlKkU3fNtgeE4FTmMSorOZT94sycUfvBMWig==;U2FsdGVkX18MIdzVE4KsKHgrhuS5QdfrSe+0ft4jlDZZcmJfrR0AT7RuRf8TPeFDw3R8qUNuwuZVrkZPJSDBgQ=="
  SECRET_DOCUMENTER_KEY: "S8anhJOWaY9lHcH/G1W5QI80qOg6oR0boF7LdbseDhb8PpMjKWrWA0s2FEqcxq1euftXpgKf3Q434bJvwoh0iKX0DZxRDuuKcQ7UDDqskOxcznliL9yQS8cInVYXir02iAnDROYQbpD2r5Ek4W9AxZUy6/mPJJkfZYz8W0b1SSPMpzsq88FkF4h4/ZxXiXLWluwQT+lyknExQ487wj2S5CnWszIRVJDX/eHQLW/WCOAh9gQsoTj/Xu+CfI1/TqwbTcpiceo5WuPz7CZ1cJolRW68CSuwnRv7q5MBrOytSdtvHu7E9CHk3WFCYlnLN5S8i4SMtg5NmYYqVmz/oUn9ag==;U2FsdGVkX18zXT1707mRUuq6XpFYflo+ipW/gjoNB5r6rS+AR3431nlCE3FVq5NvARfPqptiiZfvmPfB7AvF0RUgA0gnYEpA3Ma8Km1jURvCGZ0jSS6AXJ88Iss3H5T7PSOlDfhOBTEWeL1Tz8BL64cCBSUMrhgznYNoa77Onz6XZHBnkWGNF9FGQaoMa8wQ+zTTwb6qCgY/q6627F3DHVgp38G67psVY0zpYCMkwkvZDrlexKfbhFNbUoyTJqcdzRAameKmAOiB9C4uoWaNIZvTxJ7Ce+cHDJDpEG97enIYQn9uQJr8zr0wb4UYcvjMiucc8JjU88Z2kwFiokMiOLOEfuMZGIdGQ73T9TdGnSjDEGYKFZ3sgiqowWK7AGpm+O/wKvgsNMz+GbgSvM0eZ7MwGPGDYQdk46amsLjfPTYEw3SVQKFjjc7VBY/p4vK6kKYMzBqYUaDfiQKDqCvt87uvk6nNeViTGs14k/kNSbqQeQog3aZq7UIkxAWSZOqwSn/C2PukuXam2YtE7xOBFvOBg10+UwmDzywbPgdfMRTBXCLUCZjofVJ5UWFXC+tNim2kz095p9t/1toK3db63PMQhoQOJ7+bN71+d+lw41YRQg1ntFET9DNSNs9Fh1iqYZquI6JutnwYkj/V6suzoiXVmONMHGjPtH1OCRW3fkhRpFX77tgJ5fUjx+4mVfoCc8mHpvaGx+46a4UBV2bcksFNRevTbO9OhkJ4SSl4PGkpIE2Iek64zJePFLMkAgajE/bcHRh1lEtA8kNqUWu3PqOc1CqOyZYk0Npy68IyriSfeUjJWD3sNELawg2k12jBukYgvu1vLjPfu4z3qvzTULIvWf+3gRNrrrveN4QMpLbD8H46rRl9s6EbPIjEAiwVvO0gjLJ8Nl1lILkZLHewvS9M9yjoi3mbt5kYhQuph1Ey+tI3ldQsjDhoePxCe1uOd+11qoqRGncYDUJKhHEQoakyL6/3z3cbQ6s6lYWimbHDwks/ETshADS5QXWEUaQ8epQWpeHiQtVv9UiWCrDZP2q+dCfVlvmP+Drqo2IRzPenuJu9uy0oT+mSwL3mY83rYvOSBbQBbO9/zTk1nUCOX0KElWRWO/no0IyE8qmJylJ2zhST1kfKBe9qIyb/ed35cxXdOl2x5u6aTHgdEdaqsg7AhXn37TlQvIG0USxPPcbRNWlT2VqiJacC+/WZexxcI4/Vp0BENDv9rh1OE1OEEqfm4X0Y9rZ9nMbKUSQ8zmNJsaDEgrFdWtIRiIB7H5oJtFN4nVAHjo81OwoaqT2fa9oVy2JvZq24j3W+fzhqaIyiT85S/xz71o0ahtV9xjqLzTBIRGBuwP1K8X4eLoDx/RUpeqcuhVnCPHotBWIsxVGPV/n9RHzjSMcC1KIwzR0OiN2tRBnbrkeYKqKhptbVWndSXXzeD51THkUDiz1VEUyVwzaRhHQkSNUXKzzai+Gc21/jpRtkr7v3laLMqhy/3oWD8S+bTMl/t/Wz3B3fgfJjCUqjQg9EvOvVZ1pEWtSnCjnltExP6mR7yabsoB+NbHiUoynLJ/n5kNqUtQ5oRevAX4WMyRBGRrcGOFP/4bhHN/+7ikrIXYmBC8KLlJQZm2VOvZfXc5xSgFoLebXb5xbzmph50hw3erQMeMx7nmNlIhDg2EixcajcduT2BjQT/if+oifu9SopZyCSdtytO/x3PCVDS4H5+yOUZ8jCFW/LRNYo6uI7YKht2SiCwccNcA4vwlSnlWWjgyvEoZp074NdM8NljL9U1WZcdbLcVcgJ69L+0IwaUfuz3x6GHr12Yxvn+LGbk18BMpkdpF06M77KxFrOK0R843wwQ82Ipq78PTB0q3LhKoCBK5pL+0JFq4ZXOkJMJtqeDJhR4tM+hBX2l5sTPyYCZHrPDuhSZhurw4DhhbUpr059hZyc/MftDnQeb2Hw++GHd6HpRSkiLs8y0hSJkIJJxvMQg73j129t8/GJC2YISVff9QEIfwdC/Qo4cpuVaxvRGDXFcMm86gprwVPOdL64p1xla36hWZF4E7S0dvAgLeFUFVOjVIYtn4QsXcqoyGXLiCiKKNq+pS3eILeqcTVSgtUpoeFtAXtJgcm6c0YzRHwMOvW+pFIGM1D9zri60gLFecK3CjaBG0pgfIZdjtMHrCfUnX/IdZWY+gZskbaJaTGjm6sTBF2rIdJQsTymIP1dUJvWzagMNvZRbYykxVoMQM54w7d5x8ROHuOPDbC4DRkzzpxtK68mk+fL66NlYWn1fBKWXdgNE1sYtEm1l3eixgnmBGYNzrWZdsXp/M9Yl23ygZpu4Wj1VZJXw/SrjJgtmaPh0M56oBfXFNSifmUtK85y8j8fbmepY+r0yOwqsVStlXHWBhN8p2XvlrRHbPtGM+Vbeo/SCpFMN9/ssmckbieJO5/yIkXRWPCPJlqiCyF584k0ZYwGcdDqQFf8h8JwTQcE3C+o5LTPpiXAOteVyQEiiu7MHZjhRcFxHF8Tzz0HbRa4rC0DXhOlG9LYAFBGET9KjXB/lEVwr/Rx9tm2/X7uD9uiWxjKkg8dJVxPXf6/2Bu6vm3GsnKH0CFcWdtDZjCCCIFOd2cbhQaEcppA4BZz97fZqc7IIGTOEiO0mPLfFQ1vsOYrJ7XRe3ZWlEJ/4DNR8LrpNCInmTVudl6vfyhs10pTvmBnyWffEfma748hsl5h35BFroYV+FYp77LtTTSsGpFOzgphPmthfW6OmrOS3rYxdMx8G9RIqwWiOdc/bSffOH8IuXejh2Tx6s1sQbHDt//q8ty3ftQnN+rQ6TsGv15tYytj6+qwGgZWEn8CGeappB6H8aA9xs0aRBOYJWo+FJ9oPut8F8QtzulYIl2rRt8GYn/RTGrifwnHcI5tYKqx/LxKaZhHxzedUxNOLiH19FcpQsw6+MZJEYC7kmN4NoBDVzhFgIywwpDuLkB0nXLkasP2IqYOgV1y8wuJXk+VIcIFuoPWnbzJMjScH5PF4dWJidbY4UXWKsUup/8lgeABsPzVgCe1e7OLEiPqa/cqTn9DdOKoTk/qEAQttU6+j2cUULEnpxIrSnYe+xDPl8MEXE86zJse6l06ct2kjPVaU+mAVWkQMJ+htoz4xFyjXWbmAuh0MOkcGIjKCqNzPWwfOPjam9/hqo63LFvfMhtQns2iZ+5QHADA0tX7eDgiBkKqOaZt3TdexqRal2NXv5dvEmVBt6GA07+xfLH0+XHsRkXBfvVWvCldobmzd3Rl50gK3r6Yptr6g5Vh+hY0cWOm8tHykcQvUPma7O/WRghe+9rIRC6Qto/uKjwBgRnrSJjMVg0NqEj/ckpZYpaSImAwNQwcWZIzjBCGamzFKSVZrKbmUTNIfkC1/arWgsmeE/73eduiczN7luZbseIBKHzJTFahlJ7125CQeCaGXWzbQOBGU8lBN+UQL9RLbIJxFS28OlU5yZb92bep25e4unSFINY+uW+fi18ZOAYw+W4krbnFKgTIZHV16nYugPXbnKk0JwirU4H0jhA4TclwG1t8KOrPLCKU2wMr2CiTkgx2FPeu06eX8Eix/9bn964aJhvQt+Lz/ljWZLlWykWyaiLeP3mEHERujJVDzR7aAPmZ1t7lmWtULRGRiqWuLXD4W1Wa34zr6H1LrFI4tkokngaWusuq3tcjyuyM5pKyzCCpKXQAVvxuIRu6E5/Qr/n8UaNZfjp233RPjY1EYMkMKDLbtYABcZa2S9YmIEgikCnJd5PEDovPAB3Uez/DdOVqz5O3hgUxapQlKImgfmEHtm82Zl4txhRVTdQh3dGsvUgH0UkUl0rMd2BOWz0Edtb+yG1cJu2eZ13pkQEn3QDAPOnNIh54iDLf4Il03mSia9QMM4TM+9+daQVwus/KlkovW1IdE7TJY8xcW/5JHFAdjWCa1BurE++nRmM5y5o2AbFcYXq2WwkWp2WLZ8zzl6uG9UU8Y8Hps5ajCL47KeBrl42BWaxogG64DG+vxgNuFYNoeSxCpYwrsG50KaAvorWUF8webiSJI90T9R9WKIXCYOwmpZyWtErJvBThzl9ZRXaxznUyw06ty5Blh06a6CB/DUhQx4cCXl3HgOcCtsIcL0RAnjE+PZUoC4EIIgFGnwi3eQpoggNasdV3M3nQfFDCxtsYofZayc7cQLryfd55N8ddwAtE/HXOtWRif/5Q+lkhzbLz0e5jKM+yEnQWokjQkqnjI/MWlbsTgv1tAAG0EFr8xDQx28gzAnkHBmP5uZOfbEh5gp90iIHL5H7s+6de1pZNmB6eG4Z+108IkBzJvOcKKHyGwDCTq5MshPBHmJ1w19vYX94LTLMpKF+ML6Few7akgbmGxN/ZiMfc5gSEb+BN77FuA7RnKoj6o8cqUM8SX/rFg6XpWhnqqigMtSHuSi0b1SzqvpHkNxElIDa/J55giQF117Igu7Z27BZadjmvc2JBCDQhh6a7VQ7MGIw9OqaoiV3Rckqn7T5LKGNeYWVCc4UnotbcDTr4Y3xtzFGzVi2ZqioWwepmSS54RPsmOqHSkgdGs0mlTq5mA8Y13oJjZv7toVo0eui5gbHK00PvDzEqblJA5PozXLTeZv0mvkVbkK/NzKUZ6zgGE2WW/T9VfOaotjsPYPIHlo7N1H6P0iw0yH4="
  SECRET_CODESPEED_SERVER: "n6tQqMwRA5WJZXPT9sVmKBFR+xGwoq2DMlhGIQLETkyTbvNLAgRERcMR7Udxz4nEN321Ieo7U6/vwPWeDnzhsv+Qcx9jEK0BgWvJ1iDyPYxw/x2yrrP4OnFyEyl0sydHjpDSMakvSCqu5E3YW0J0eN2eSVg0BtoNqSD7BCRh3Fh2lHrEVD8FjZLhdR2lSepJBwWhN3QjK+2RsNEMJJKAQTimDJxqPr+9UDNr1Zhiy2Yn3H4aV4TZ6XX2ioCVTG7rI3VjFjFWNJ6kwQIW0MEcF56OIhnuNEgGNIvegsrkhitKvWUxeeuYEY6imfSvKm2Ljyw5RY3UHvqxg82hrWx2tw==;U2FsdGVkX19Pf6SMbEv9EN/rv6GwO3NMzFSjoE4G5FNaGMLyKOieku7o2jCFju46wt4hF7KvpHXZFAr0azbT42nWZromLvmrrZnumUuVW98="