rebench_0.6.conf

#ReBench -*- mode: yaml; yaml-indent-offset: 4 -*-

standard_experiment: CrossBenchmarks
standard_data_file: 'output/pycket.data'

reporting:
    # results can also be reported to a codespeed instance
    # see: https://github.com/tobami/codespeed
    # codespeed:
    #     url: http://localhost:1/ # not supposed to work 
    #     # other details like commitid are required to be given as parameters
    confidence_level: 0.95
    # irc:
    #     server:  chat.freenode.net
    #     port:    6666
    #     channel: '#pycket'
    #     nick:    rebenchbot
    #     log_events:
    #         - job_completed
    #         #- run_failed

runs:
    number_of_data_points: 10

# settings for quick runs, useful for fast feedback during experiments
quick_runs:
    number_of_data_points: 3
    # max_time: 60   # time in seconds
    max_time: 300000   # time in seconds
################################################################
# definition of benchmark suites
benchmark_suites:
    CrossBenchmarks:
        gauge_adapter: Multivariate
        location: CrossBenchmarks
        command: -%(variable)s %(benchmark)s
        #variable_values: [nothing, fixflo, unsafe]
        variable_values: [nothing, fixflo]
        benchmarks: &defaultbenchmarks
            - ack
            - array1
            - browse
            - cat
            - conform
            - cpstak
            - ctak
            - dderiv
            - deriv
            - destruc
            - diviter
            - divrec
            - dynamic
            - earley
            - fft
            - fib
            - fibc
            - fibfp
            - gcbench
            - gcold
            - graphs
            - lattice
            - matrix
            - maze
            - mazefun
            - mbrot
            - nqueens
            - nucleic
            - paraffins
            - parsing
            - perm9
            - peval
            - pi
            - pnpoly
            - primes
            - puzzle
            - ray
            - scheme
            - simplex
            - slatex
            - string
            - sum
            - sum1
            - sumfp
            - sumloop
            - tail
            - tak
            - takl
            #- tfib
            - trav1
            - trav2
            - triangl
            - wc
            - boyer
            - sboyer
            - nboyer
    CrossBenchmarksNothing:
        gauge_adapter: Multivariate
        location: CrossBenchmarks
        command: -%(variable)s %(benchmark)s
        variable_values: [nothing]
        benchmarks: *defaultbenchmarks
    ShootoutBenchmarks:
        gauge_adapter: Multivariate
        location: ShootoutBenchmarks
        command: PURE %(benchmark)s
        benchmarks:
            - ackermann:
                extra_args: 12
            - ary:
                extra_args: 1700000
            - binarytrees:
                extra_args: 17
            - binarytrees-generic:
                extra_args: 17
            # - chameneos:
            #     extra_args: 120000
            # - cheapconcurrency:
            #     extra_args: 2000
            # - echo: # needs thread
            #     extra_args: 200000
            # - except: # call-with-continuation-prompt
            #     extra_args: 2000000
            - fannkuch:
                extra_args: 10
            - fannkuch-redux:
                extra_args: 10
            - fannkuch-redux-generic:
                extra_args: 10
            - fasta:
                extra_args: 100000000
            - fasta-generic:
                extra_args: 100000000
            - fibo:
                extra_args: 42
            - hash:
                extra_args: 4000000
            - hash2:
                extra_args: 6000
            - heapsort:
                extra_args: 1500000
            # - k-nucleotide:
            #     extra_args: fasta-500000
            - lists:
                extra_args: 2000
            - mandelbrot:
                extra_args: 5000
            - mandelbrot-generic:
                extra_args: 5000
            - matrix:
                extra_args: 25000
            - meteor:
                extra_args: 200000
            - moments:
                extra_args: moments-2000
            - nbody:
                extra_args: 8000000
            - "nbody-generic":
                extra_args: 8000000
            - "nbody-vec":
                extra_args: 15000000
            - "nbody-vec-generic":
                extra_args: 15000000
            - nestedloop:
                extra_args: 35
            - nsieve:
                extra_args: 12
            - nsievebits:
                extra_args: 12
            - partialsums:
                extra_args: 15000000
            - pidigits:
                extra_args: 7000
            - pidigits1:
                extra_args: 20000
            - random:
                extra_args: 250000000
            - recursive:
                extra_args: 12
            - regexmatch:
                extra_args: regexmatch-1000000
            # - regexpdna:
            #     extra_args: fasta-1000000
            # - reversecomplement:
            #    extra_args: fasta-30000000
            # - reversecomplement-generic:
            #    extra_args: fasta-30000000
            # - reversefile:
            #     extra_args: sumcol-3000
            - sieve:
                extra_args: 50000
            - spellcheck:
                extra_args: ulysses # or jabberwock_hunting, alice
            - spectralnorm:
                extra_args: 3000
            - "spectralnorm-generic":
                extra_args: 3000
            - strcat:
                extra_args: 110000000
            - sumcol:
                extra_args: sumcol-20000
            # - "thread-ring":
            #     extra_args: 1000000
            - wc:
                extra_args: sumcol-20000
            - wordfreq:
                extra_args: sumcol-20000
    ChaperoneProcBenchmarks:
        gauge_adapter: Multivariate
        location: ChaperoneBenchmarks
        command: "CHAP proc-bm-%(benchmark)s"
        benchmarks:
            - direct
            - indirect
            - wrapped
            - wrapped+check
            - wrapped+return
            - proxy
            - impersonate
            - chaperone
            - impersonate+return
            - chaperone+return
    ChaperoneChurchBenchmarks:
        gauge_adapter: Multivariate
        location: ChaperoneBenchmarks
        command: "CHAP church-%(benchmark)s"
        benchmarks:
            - direct
            - wrap
            - proxy
            # - proxy-a
            - chaperone
            # - chaperonea
            - contract
    ChaperoneStructBenchmarks:
        gauge_adapter: Multivariate
        location: ChaperoneBenchmarks
        command: "CHAP struct-bm-%(benchmark)s"
        benchmarks:
            - direct
            - impersonate
            - chaperone
            - unsafe
            - unsafe2
            - proxy
    ChaperoneBubbleBenchmarks:
        gauge_adapter: Multivariate
        location: ChaperoneBenchmarks
        command: "CHAP bubble-%(benchmark)s"
        benchmarks:
            - direct
            - proxy
            - chaperone
            - contract
            - unsafe
            - unsafe2
    ChaperonePffBenchmarks:
        gauge_adapter: Multivariate
        location: ChaperoneBenchmarks/src/struct
        command: "CHAPM pff.rkt --stderr koala-face.trace '%(benchmark)s' 0"
        # variable_values: [0, 1, 2]
        benchmarks:
            - "none"
            - "opt chap"
    ChaperoneOdeBenchmarks:
        gauge_adapter: Multivariate
        location: ChaperoneBenchmarks
        command: "CHAP ode-%(benchmark)s"
        benchmarks:
            - direct
            - contract


################################################################

virtual_machines:
    Pycket:
        path: bin
        binary: run-pycket
    Racket:
        path: bin
        binary: run-racket
    Larceny:
        path: bin
        binary: run-larceny
    Gambit:
        path: bin
        binary: run-gambit
    Bigloo:
        path: bin
        binary: run-bigloo
    V8:
        path: bin
        binary: run-v8
    Spidermonkey:
        path: bin
        binary: run-spidermonkey
    Python:
        path: bin
        binary: run-python
    Pypy:
        path: bin
        binary: run-pypy
    PycketNoCallgraph:
        path: bin
        binary: run-pycket-no-callgraph
    PycketNoStrategies:
        path: bin
        binary: run-pycket-no-strategies
    PycketNoPruneEnv:
        path: bin
        binary: run-pycket-no-prune-env
    PycketNoTypeSizeSpecialization:
        path: bin
        binary: run-pycket-no-type-size-specialization
    PycketNoJit:
        path: bin
        binary: run-pycket-nojit
        number_of_data_points: 1

################################################################
experiments:
    #
    # Cross-Scheme Benchmarks
    #
    CrossBenchmarks:
        description: >
            All the portable, cross-dialect benchmarks from Larceny
        benchmark: CrossBenchmarks
        data_file: output/CrossBenchmarks.tsv
        number_of_data_points: 1
        executions: &crossvms
          - Racket
          - Pycket
          - Gambit
          - Larceny
          - Bigloo
    CrossBenchmarkFast:
        benchmark: CrossBenchmarks
        data_file: output/CrossBenchmarks_fast.tsv
        number_of_data_points: 1
        executions: *crossvms
    CrossBenchmarksPycket:
        benchmark: CrossBenchmarks
        data_file: output/CrossBenchmarks_pycket.tsv
        executions: Pycket
    CrossBenchmarksPycketFast:
        benchmark: CrossBenchmarks
        number_of_data_points: 1
        data_file: output/CrossBenchmarks_pycket_fast.tsv
        executions: Pycket
    CrossBenchmarksOther:
        benchmark: CrossBenchmarks
        data_file: output/CrossBenchmarks_other.tsv
        executions: &crossother
          - Racket
          - Gambit
          - Larceny
          - Bigloo
    CrossBenchmarksOtherFast:
        benchmark: CrossBenchmarks
        number_of_data_points: 1
        data_file: output/CrossBenchmarks_other_fast.tsv
        executions: *crossother
    CrossBenchmarksRacketPycketFast:
        benchmark: AllBenchmarks
        number_of_data_points: 1
        data_file: output/CrossBenchmarks_fast_racket_pycket.tsv
        executions:
            - Pycket
            - Racket
    #
    # Shootout benchmarks (Racket/Pycket)
    #
    ShootoutBenchmarks:
        description: Run the Racket shootout benchmarks
        benchmark: ShootoutBenchmarks
        data_file: output/Shootout.tsv
        executions:
            - Pycket
            - Racket
    ShootoutBenchmarksRacket:
        benchmark: ShootoutBenchmarks
        data_file: output/Shootout_racket.tsv
        executions: Racket
    ShootoutBenchmarksRacketFast:
        benchmark: ShootoutBenchmarks
        data_file: output/Shootout_racket_fast.tsv
        number_of_data_points: 1
        executions: Racket
    ShootoutBenchmarksPycket:
        benchmark: ShootoutBenchmarks
        data_file: output/Shootout_pycket.tsv
        executions: Pycket
    ShootoutBenchmarksPycketFast:
        benchmark: ShootoutBenchmarks
        number_of_data_points: 1
        data_file: output/Shootout_pycket_fast.tsv
        executions: Pycket
    ShootoutBenchmarksFast:
        benchmark: ShootoutBenchmarks
        number_of_data_points: 1
        executions:
            - Pycket
            - Racket
        data_file: output/Shootout_fast.tsv
    #
    # Chaperone Benchmarkset
    #
    ChaperoneMicroBenchmarks:
        description: Run the Racket chaperone benchmarks
        benchmark: &chapmicrobenches
            - ChaperoneProcBenchmarks
            - ChaperoneChurchBenchmarks
            - ChaperoneStructBenchmarks
            - ChaperoneBubbleBenchmarks
            # - ChaperoneMacroBenchmarks
        # benchmark: ChaperoneMicroBenchmarks
        data_file: output/ChaperoneMicro.tsv
        executions: &chapvms
            # - Python
            - Pypy
            - Pycket
            - PycketNoCallgraph
            - Racket
            - Larceny
            - V8
            # - Spidermonkey
    ChaperoneMicroBenchmarksPycket:
        benchmark: *chapmicrobenches
        data_file: output/ChaperoneMicro_pycket.tsv
        executions:
            - Pycket
            - PycketNoCallgraph
    ChaperoneMicroBenchmarksPycketFast:
        benchmark: *chapmicrobenches
        number_of_data_points: 1
        data_file: output/ChaperoneMicro_pycket_fast.tsv
        executions:
            - Pycket
            - PycketNoCallgraph
    ChaperoneMicroBenchmarksOther:
        benchmark: *chapmicrobenches
        data_file: output/ChaperoneMicro_other.tsv
        executions: &chapothervms
            # - Python
            - Pypy
            - Racket
            - Larceny
            - V8
            # - Spidermonkey
    ChaperoneMicroBenchmarksOtherFast:
        benchmark: *chapmicrobenches
        number_of_data_points: 1
        data_file: output/ChaperoneMicro_other_fast.tsv
        executions: *chapothervms
    # Chaperone Macro
    ChaperoneMacroBenchmarks:
        description: Run the Racket chaperone benchmarks
        benchmark: &chapmacrobenches
            - ChaperonePffBenchmarks
            - ChaperoneOdeBenchmarks
        data_file: output/ChaperoneMacro.tsv
        executions: &chapmacvms
            - Pycket
            - PycketNoCallgraph
            - Racket
    ChaperoneMacroBenchmarksPycketFast:
        benchmark: *chapmacrobenches
        number_of_data_points: 1
        data_file: output/ChaperoneMacro_pycket_fast.tsv
        executions:
            - Pycket
            - PycketNoCallgraph
    ChaperoneMacroBenchmarksRacketFast:
        benchmark: *chapmacrobenches
        number_of_data_points: 1
        data_file: output/ChaperoneMacro_other_fast.tsv
        executions: Racket
    ChaperoneMacroBenchmarksPycket:
        benchmark: *chapmacrobenches
        data_file: output/ChaperoneMacro_pycket.tsv
        executions:
            - Pycket
            - PycketNoCallgraph
    ChaperoneMacroBenchmarksRacket:
        benchmark: *chapmacrobenches
        data_file: output/ChaperoneMacro_other.tsv
        executions: Racket
    #
    # Pycket comparison
    #
    ComparePyckets:
        descritption: >
            Compare the differently optimized pyckets
        data_file: output/PycketVsPycket.tsv
        benchmark: &pycketbenches
            - CrossBenchmarksNothing
            - ShootoutBenchmarks
        executions: &pycketvms
            - Pycket
            - PycketNoCallgraph
            - PycketNoStrategies
            - PycketNoPruneEnv
            - PycketNoTypeSizeSpecialization
    ComparePycketsFast:
        number_of_data_points: 1
        data_file: output/PycketVsPycket_fast.tsv
        benchmark: *pycketbenches
        executions: *pycketvms
# EOF