Run multi-thread benchmarks in CI

.travis.yml

@@ -38,8 +38,7 @@ jobs:
         - julia -e 'using Run; Run.prepare("test/environments/jl10")'
       script:
         - julia -e 'using Run; Run.test(project="test/environments/jl10")'
-    - name: "Benchmark"
-      julia: 1.1
+    - name: "Benchmark (Sequential)"
       os: linux
       install:
         - julia -e 'using Run; Run.prepare("benchmark")'
@@ -52,6 +51,19 @@ jobs:
         - julia -e 'using Run; Run.script("benchmark/pprintjudge.jl")'
       after_success: skip
       if: NOT (branch = master)
+    - name: "Benchmark (Multi-thread)"
+      os: linux
+      install:
+        - julia -e 'using Run; Run.prepare("benchmark")'
+      before_script:
+        - git fetch origin '+refs/heads/master:refs/remotes/origin/master'
+        - git branch master origin/master
+        # Run benchmark outside `script` so that it's hidden by default:
+        - julia -e 'using Run; Run.script("benchmark/runjudge_multi_thread.jl")'
+      script:
+        - julia -e 'using Run; Run.script("benchmark/pprintjudge_multi_thread.jl")'
+      after_success: skip
+      if: NOT (branch = master)
     - stage: "Documentation"
       julia: 1.3
       os: linux

benchmark/bench_words.jl → benchmark/multi-thread/bench_words.jl

@@ -1,6 +1,6 @@
 module BenchWords
 using BenchmarkTools
-include("../examples/words.jl")
+include("../../examples/words.jl")
 
 suite = BenchmarkGroup()
 

benchmark/multi-thread/benchmarks.jl

@@ -0,0 +1,16 @@
+if lowercase(get(ENV, "CI", "false")) == "true"
+    @info "Executing in CI. Instantiating benchmark environment..."
+    using Pkg
+    Pkg.activate(dirname(@__DIR__))
+    Pkg.instantiate()
+end
+
+using BenchmarkTools
+SUITE = BenchmarkGroup()
+for file in readdir(@__DIR__)
+    file == "bench_words.jl" && continue
+    if startswith(file, "bench_") && endswith(file, ".jl")
+        SUITE[file[length("bench_") + 1:end - length(".jl")]] =
+            include(file)
+    end
+end

benchmark/pprintjudge_multi_thread.jl

@@ -0,0 +1,14 @@
+using PkgBenchmark
+include("pprinthelper.jl")
+benchdir = joinpath(@__DIR__, "multi-thread")
+group_target = PkgBenchmark.readresults(joinpath(benchdir, "result-target.json"))
+group_baseline = PkgBenchmark.readresults(joinpath(benchdir, "result-baseline.json"))
+judgement = judge(group_target, group_baseline)
+
+displayresult(judgement)
+
+printnewsection("Target result")
+displayresult(group_target)
+
+printnewsection("Baseline result")
+displayresult(group_baseline)

benchmark/runjudge_multi_thread.jl

@@ -0,0 +1,36 @@
+using PkgBenchmark
+
+mkconfig(; kwargs...) =
+    BenchmarkConfig(
+        env = Dict(
+            "JULIA_NUM_THREADS" => "2",
+        );
+        kwargs...
+    )
+
+progressoptions =
+    if lowercase(get(ENV, "CI", "false")) == "true"
+        (dt = 60 * 9.0,)
+    else
+        NamedTuple()
+    end
+
+benchdir = joinpath(@__DIR__, "multi-thread")
+
+group_target = benchmarkpkg(
+    dirname(@__DIR__),
+    mkconfig(),
+    script = joinpath(benchdir, "benchmarks.jl"),
+    progressoptions = progressoptions,
+    resultfile = joinpath(benchdir, "result-target.json"),
+)
+
+group_baseline = benchmarkpkg(
+    dirname(@__DIR__),
+    mkconfig(id = "master"),
+    script = joinpath(benchdir, "benchmarks.jl"),
+    progressoptions = progressoptions,
+    resultfile = joinpath(benchdir, "result-baseline.json"),
+)
+
+judgement = judge(group_target, group_baseline)