performance-benchmarks-kurobako.yml

name: Performance Benchmarks with kurobako

on:
  workflow_dispatch:
    inputs:
      sampler-list:
        description: 'Sampler List: A list of samplers to check the performance. Should be a whitespace-separated list of Optuna samplers. Each sampler must exist under `optuna.samplers` or `optuna.integration`.'
        required: false
        default: 'RandomSampler TPESampler'
      sampler-kwargs-list:
        description: 'Sampler Arguments List: A list of sampler keyword arguments. Should be a whitespace-separated list of json format dictionaries.'
        required: false
        default: '{} {\"multivariate\":true,\"constant_liar\":true}'
      pruner-list:
        description: 'Pruner List: A list of pruners to check the performance. Should be a whitespace-separated list of Optuna pruners. Each pruner must exist under `optuna.pruners`.'
        required: false
        default: 'NopPruner'
      pruner-kwargs-list:
        description: 'Pruner Arguments List: A list of pruner keyword arguments. Should be a whitespace-separated list of json format dictionaries.'
        required: false
        default: '{}'
      budget:
        description: 'Number of Trials if the pruning is not enabled. If the pruning is enabled, the total number of steps is equal to `budget * (steps per trial)`.'
        required: false
        default: '80'
      n-runs:
        description: 'Number of Studies'
        required: false
        default: '10'
      n-concurrency:
        description: 'Number of Concurrent Trials'
        required: false
        default: '1'


jobs:
  performance-benchmarks-kurobako:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: Setup Python3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Install gnuplot
      run: |
        sudo apt update
        sudo apt -y install gnuplot

    - name: Setup cache
      uses: actions/cache@v3
      env:
        # Caches them under a common name so that they can be used by other performance benchmark.
        cache-name: performance-benchmarks
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-3.9-${{ env.cache-name }}-${{ hashFiles('**/pyproject.toml') }}-v1
        restore-keys: |
          ${{ runner.os }}-3.9-${{ env.cache-name }}-${{ hashFiles('**/pyproject.toml') }}

    - name: Install Python libraries
      run: |
        python -m pip install --upgrade pip
        pip install --progress-bar off -U setuptools

        # Install minimal dependencies and confirm that `import optuna` is successful.
        pip install --progress-bar off .
        python -c 'import optuna'
        optuna --version

        pip install --progress-bar off .[benchmark] --extra-index-url https://download.pytorch.org/whl/cpu

        pip install --progress-bar off kurobako

    - name: Cache kurobako CLI
      id: cache-kurobako
      uses: actions/cache@v3
      with:
        path: ./kurobako
        key: kurobako-0-2-9

    - name: Install kurobako CLI
      if: steps.cache-kurobako.outputs.cache-hit != 'true'
      run: |
        curl -L https://github.com/optuna/kurobako/releases/download/0.2.9/kurobako-0.2.9.linux-amd64 -o kurobako

        chmod +x kurobako
        ./kurobako -h

    - name: Output installed packages
      run: |
        pip freeze --all
    - name: Output dependency tree
      run: |
        pip install pipdeptree
        pipdeptree

    - name: Cache hpobench dataset
      id: cache-hpobench-dataset
      uses: actions/cache@v3
      with:
        path: ./fcnet_tabular_benchmarks
        key: hpobench-dataset

    - name: Download hpobench dataset
      if: steps.cache-hpobench-dataset.outputs.cache-hit != 'true'
      run: |
        wget http://ml4aad.org/wp-content/uploads/2019/01/fcnet_tabular_benchmarks.tar.gz
        tar xf fcnet_tabular_benchmarks.tar.gz

    - name: Cache nasbench dataset
      id: cache-nasbench-dataset
      uses: actions/cache@v3
      with:
        path: ./nasbench_full.bin
        key: nasbench-dataset

    # Ref: https://github.com/optuna/kurobako/wiki/NASBench
    - name: Download nasbench dataset
      if: steps.cache-nasbench-dataset.outputs.cache-hit != 'true'
      run: |
        curl -L $(./kurobako dataset nasbench url) -o nasbench_full.tfrecord
        ./kurobako dataset nasbench convert nasbench_full.tfrecord nasbench_full.bin

    - name: Run performance benchmark
      run: |
        python benchmarks/run_kurobako.py \
          --path-to-kurobako "." \
          --name-prefix "" \
          --budget ${{ github.event.inputs.budget }} \
          --n-runs ${{ github.event.inputs.n-runs }} \
          --n-jobs 10 \
          --n-concurrency ${{ github.event.inputs.n-concurrency }} \
          --sampler-list '${{ github.event.inputs.sampler-list }}' \
          --sampler-kwargs-list '${{ github.event.inputs.sampler-kwargs-list }}' \
          --pruner-list '${{ github.event.inputs.pruner-list }}' \
          --pruner-kwargs-list '${{ github.event.inputs.pruner-kwargs-list }}' \
          --seed 0 \
          --data-dir "." \
          --out-dir "out"

    - uses: actions/upload-artifact@v3
      with:
        name: benchmark-report
        path: |
          out/report.md
          out/**/*.png

    - uses: actions/download-artifact@v2
      with:
        name: benchmark-report
        path: |
          out/report.md
          out/**/*.png