config.yml

version: 2.1

# -------------------------------------------------------------------------------------
# Commands
# -------------------------------------------------------------------------------------

commands:

  py_3_7_setup:
    description: "Install and switch to Python 3.7.5; also install pip and pytest."
    steps:
      - run:
          name: "Setup Python v3.7.5 environment"
          command: |
            cd /opt/circleci/.pyenv && git pull && cd -
            pyenv install -s 3.7.5
            pyenv global 3.7.5
            pyenv local 3.7.5
            pyenv versions
            echo "In venv: $(pyenv local) - $(python -V), $(pip -V)"
            sudo "$(which python)" -m pip install --upgrade pip
            sudo "$(which python)" -m pip install pytest

  run_nvidia_smi:
    description: "Prints GPU capabilities from nvidia-smi"
    steps:
      - run:
          name: "Run Nvidia-SMI"
          command: |
            nvidia-smi

  pip_dev_install:
    description: "Install dependencies via pip, including extra deps. Also supports more options, such as building on top of PyTorch nightly."
    parameters:
      args:
        type: string
        default: ""
    steps:
      - run:
          name: "Install dependencies via pip"
          command: ./scripts/install_via_pip.sh << parameters.args >>

  lint_flake8:
    description: "Lint with flake8"
    steps:
      - run:
          name: "Lint with flake8"
          command: flake8 --config ./.circleci/flake8_config.ini

  lint_black:
    description: "Lint with black"
    steps:
      - run:
          name: "Lint with black"
          command: black --check --diff --color .

  isort:
    description: "Check import order with isort"
    steps:
      - run:
          name: "Check import order with isort"
          command: isort -v -l 88 -o opacus --lines-after-imports 2 -m 3 --trailing-comma --check-only .

  configure_docusaurus_bot:
    description: "Configure Docusaurus GitHub bot"
    steps:
      - run:
          name: "Configure Docusaurus GitHub bot"
          command: |
              git config --global user.email "docusaurus-bot@users.noreply.github.com"
              git config --global user.name "Opacus website deployment script"
              echo "machine github.com login docusaurus-bot password $DOCUSAURUS_GITHUB_TOKEN" > ~/.netrc

  deploy_site:
    description: "Deploy website to GitHub Pages"
    steps:
      - run:
          name: "Deploy website to GitHub Pages"
            # TODO: make the installation above conditional on there being relevant changes (no need to install if there are none)
          command: |
              mkdir -p website/static/.circleci && cp -a .circleci/. website/static/.circleci/.
              cd website
              ./scripts/build_website.sh -b
              GIT_USER=docusaurus-bot yarn run publish-gh-pages


  unit_tests:
    description: "Run unit tests"
    steps:
      - run:
          name: "Unit tests & doctests"
          no_output_timeout: 1h
          command: |
            mkdir unittest-reports
            python -m pytest --doctest-modules -p conftest --junitxml=unittest-reports/junit.xml opacus

      - store_test_results:
          path: unittest-reports
      - store_artifacts:
          path: unittest-reports


  mnist_integration_test:
    description: "Runs MNIST example end to end"
    parameters:
      device:
        default: "cpu"
        type: string
    steps:
      - run:
          name: MNIST example
          command: |
            mkdir -p runs/mnist/data
            mkdir -p runs/mnist/test-reports
            echo "Using $(python -V) ($(which python))"
            echo "Using $(pip -V) ($(which pip))"
            python examples/mnist.py --lr 0.25 --sigma 0.7 -c 1.5 --batch-size 64 --epochs 1 --data-root runs/mnist/data --n-runs 1 --device <<parameters.device>>
            python -c "import torch; accuracy = torch.load('run_results_mnist_0.25_0.7_1.5_64_1.pt'); exit(0) if (accuracy[0]>0.78 and accuracy[0]<0.95) else exit(1)"
          when: always
      - store_test_results:
          path: runs/mnist/test-reports
      - store_artifacts:
          path: runs/mnist/test-reports

  mnist_lightning_integration_test:
    description: "Runs MNIST-Lightning example end to end"
    parameters:
      device:
        default: "cpu"
        type: string
    steps:
      - run:
          name: MNIST-Lightning example
          command: |
            mkdir -p runs/mnist/data
            mkdir -p runs/mnist/test-reports
            echo "Using $(python -V) ($(which python))"
            echo "Using $(pip -V) ($(which pip))"
            python examples/mnist_lightning.py fit --trainer.accelerator <<parameters.device>> --model.lr 0.25 --model.sigma 0.7 --model.max_per_sample_grad_norm 1.5 --model.sample_rate 0.004 --trainer.max_epochs 1 --data.data_dir runs/mnist/data --data.sample_rate 0.004
            python -c "import torch; exit(0)"
          when: always
      - store_test_results:
          path: runs/mnist-lightning/test-reports
      - store_artifacts:
          path: runs/mnist-lightning/test-reports

  cifar10_integration_test:
    description: "Runs CIFAR10 example end to end"
    parameters:
      device:
        default: "cpu"
        type: string
    steps:
      - run:
          name: CIFAR10 example
          command: |
            mkdir -p runs/cifar10/data
            mkdir -p runs/cifar10/logs
            mkdir -p runs/cifar10/test-reports
            echo "Using $(python -V) ($(which python))"
            echo "Using $(pip -V) ($(which pip))"
            pip install tensorboard
            python examples/cifar10.py --lr 0.1 --sigma 1.5 -c 10 --sample-rate 0.04 --epochs 10 --data-root runs/cifar10/data --log-dir runs/cifar10/logs --device <<parameters.device>>
            python -c "import torch; model = torch.load('model_best.pth.tar'); exit(0) if (model['best_acc1']>0.4 and model['best_acc1']<0.49) else exit(1)"
          when: always
      - store_test_results:
          path: runs/cifar10/test-reports
      - store_artifacts:
          path: runs/cifar10/test-reports

  dcgan_integration_test:
    description: "Runs dcgan example end to end"
    parameters:
      device:
        default: "cpu"
        type: string
    steps:
      - run:
          name: dcgan example
          command: |
            mkdir -p runs/dcgan/data
            mkdir -p runs/dcgan/test-reports
            echo "Using $(python -V) ($(which python))"
            echo "Using $(pip -V) ($(which pip))"
            python examples/dcgan.py --lr 2e-4 --sigma 0.7 -c 1.5 --batch-size 32 --epochs 1 --data-root runs/dcgan/data --device <<parameters.device>>
          when: always
      - store_test_results:
          path: runs/dcgan/test-reports
      - store_artifacts:
          path: runs/dcgan/test-reports

  imdb_integration_test:
    description: "Runs imdb example end to end"
    parameters:
      device:
        default: "cpu"
        type: string
    steps:
      - run:
          name: imdb example
          command: |
            mkdir -p runs/imdb/data
            mkdir -p runs/imdb/test-reports
            echo "Using $(python -V) ($(which python))"
            echo "Using $(pip -V) ($(which pip))"
            pip install --user datasets transformers
            python examples/imdb.py --lr 0.02 --sigma 0.56 -c 1.0 --batch-size 32 --max-sequence-length 256 --epochs 1 --data-root runs/imdb/data --device <<parameters.device>>
            python -c "import torch; accuracy = torch.load('run_results_imdb_classification.pt'); exit(0) if (accuracy>0.54 and accuracy<0.66) else exit(1)"
          when: always
      - store_test_results:
          path: runs/imdb/test-reports
      - store_artifacts:
          path: runs/imdb/test-reports

  charlstm_integration_test:
    description: "Runs charlstm example end to end"
    parameters:
      device:
        default: "cpu"
        type: string
    steps:
      - run:
          name: charlstm example
          command: |
            mkdir -p runs/charlstm/data
            wget https://download.pytorch.org/tutorial/data.zip -O runs/charlstm/data/data.zip
            unzip runs/charlstm/data/data.zip -d runs/charlstm/data
            rm runs/charlstm/data/data.zip
            mkdir -p runs/charlstm/test-reports
            echo "Using $(python -V) ($(which python))"
            echo "Using $(pip -V) ($(which pip))"
            pip install scikit-learn
            python examples/char-lstm-classification.py --epochs=20 --learning-rate=2.0 --hidden-size=128 --delta=8e-5 --batch-size 400 --n-layers=1 --sigma=1.0 --max-per-sample-grad-norm=1.5 --data-root="runs/charlstm/data/data/names/" --device=<<parameters.device>> --test-every 5
            python -c "import torch; accuracy = torch.load('run_results_chr_lstm_classification.pt'); exit(0) if (accuracy>0.60 and accuracy<0.80) else exit(1)"
          when: always
      - store_test_results:
          path: runs/charlstm/test-reports
      - store_artifacts:
          path: runs/charlstm/test-reports

  benchmark_layers_integration_test:
    description: "Runs benchmark end to end"
    parameters:
      device:
        default: "cpu"
        type: string
      layers:
        default: "mha dpmha gsm_dpmha embedding gsm_embedding instancenorm gsm_instancenorm groupnorm gsm_groupnorm layernorm gsm_layernorm lstm dplstm gsm_dplstm rnn dprnn gsm_dprnn linear gsm_linear gru dpgru gsm_dpgru"
        type: string
      runtime_ratio_threshold:
        default: "7.0"
        type: string
      memory_ratio_threshold:
        default: "2.0"
        type: string
    steps:
      - run:
          name: benchmarks
          command: |
            mkdir -p benchmarks/results/raw
            echo "Using $(python -V) ($(which python))"
            echo "Using $(pip -V) ($(which pip))"
            python benchmarks/run_benchmarks.py --batch_size 16 --layers <<parameters.layers>> --config_file ./benchmarks/config.json --root ./benchmarks/results/raw/ --cont
            IFS=$' ';layers=(<<parameters.layers>>); rm -rf /tmp/report_layers; mkdir -p /tmp/report_layers; IFS=$'\n'; files=`( echo "${layers[*]}" ) | sed 's/.*/.\/benchmarks\/results\/raw\/&*/'`
            cp -v ${files[@]} /tmp/report_layers
            report_id=`IFS=$'-'; echo "${layers[*]}"`
            python benchmarks/generate_report.py --path-to-results /tmp/report_layers --save-path benchmarks/results/report-${report_id}.csv --format csv
            python benchmarks/generate_report.py --path-to-results /tmp/report_layers --save-path benchmarks/results/report-${report_id}.pkl --format pkl
            python -c "import pandas as pd; r = pd.read_pickle('./benchmarks/results/report-"$report_id".pkl').fillna(0); th="<<parameters.runtime_ratio_threshold>>"; exit(0) if (r.loc[:, ('runtime', 'dp/control')] < th).all() and (r.loc[:, ('runtime', 'gsm/control')] < th).all() else exit(1)"
            python -c "import pandas as pd; r = pd.read_pickle('./benchmarks/results/report-"$report_id".pkl').fillna(0); th="<<parameters.memory_ratio_threshold>>"; exit(0) if (r.loc[:, ('memory', 'dp/control')] < th).all() and (r.loc[:, ('memory', 'gsm/control')] < th).all() else exit(1)"
          when: always
      - store_artifacts:
          path: benchmarks/results/
# -------------------------------------------------------------------------------------
# Jobs
# -------------------------------------------------------------------------------------

jobs:

  lint_py37_torch_release:
    docker:
      - image: cimg/python:3.7.5
    steps:
      - checkout
      - pip_dev_install
      - lint_flake8
      - lint_black
      - isort

  unittest_py37_torch_release:
    docker:
      - image: cimg/python:3.7.5
    steps:
      - checkout
      - pip_dev_install
      - unit_tests

  unittest_py38_torch_release:
    docker:
      - image: cimg/python:3.8
    steps:
      - checkout
      - pip_dev_install
      - unit_tests

  unittest_py39_torch_release:
    docker:
      - image: cimg/python:3.9
    steps:
      - checkout
      - pip_dev_install
      - unit_tests

  unittest_py39_torch_nightly:
    docker:
      - image: cimg/python:3.9
    steps:
      - checkout
      - pip_dev_install:
          args: "-n"
      - unit_tests

  integrationtest_py37_torch_release_cpu:
    docker:
      - image: cimg/python:3.7.5
    steps:
      - checkout
      - py_3_7_setup
      - pip_dev_install
      - mnist_integration_test:
          device: "cpu"

  integrationtest_py37_torch_release_cuda:
    machine:
      resource_class: gpu.nvidia.small.multi
      image: ubuntu-2004-cuda-11.4:202110-01
    steps:
      - checkout
      - py_3_7_setup
      - pip_dev_install
      - run_nvidia_smi
      - benchmark_layers_integration_test:
          device: "cuda"
          layers: "groupnorm gsm_groupnorm instancenorm gsm_instancenorm layernorm gsm_layernorm dpmha"
          runtime_ratio_threshold: "2.5"
          memory_ratio_threshold: "1.6"
      - benchmark_layers_integration_test:
          device: "cuda"
          layers: "linear gsm_linear"
          runtime_ratio_threshold: "3.6"
          memory_ratio_threshold: "13.0"
      - benchmark_layers_integration_test:
          device: "cuda"
          layers: "mha gsm_dpmha"
          runtime_ratio_threshold: "3.5"
          memory_ratio_threshold: "2.0"
      - benchmark_layers_integration_test:
          device: "cuda"
          layers: "gru dpgru gsm_dpgru"
          runtime_ratio_threshold: "18.5"
          memory_ratio_threshold: "1.5"
      - benchmark_layers_integration_test:
          device: "cuda"
          layers: "lstm dplstm gsm_dplstm"
          runtime_ratio_threshold: "16.5"
          memory_ratio_threshold: "1.5"
      - benchmark_layers_integration_test:
          device: "cuda"
          layers: "rnn dprnn gsm_dprnn"
          runtime_ratio_threshold: "16.5"
          memory_ratio_threshold: "1.2"
      - benchmark_layers_integration_test:
          device: "cuda"
          layers: "embedding gsm_embedding"
          runtime_ratio_threshold: "6.0"
          memory_ratio_threshold: "15.0"
      - mnist_integration_test:
          device: "cuda"
      - cifar10_integration_test:
          device: "cuda"
      - imdb_integration_test:
          device: "cuda"
      - charlstm_integration_test:
          device: "cuda"
      - dcgan_integration_test:
          device: "cuda"

  unittest_multi_gpu:
    machine:
      resource_class: gpu.nvidia.medium.multi
      image: ubuntu-2004-cuda-11.4:202110-01
    steps:
      - checkout
      - py_3_7_setup
      - pip_dev_install
      - run_nvidia_smi
      - run:
          name: "Unit test multi_gpu"
          no_output_timeout: 1h
          command: |
            mkdir unittest-multigpu-reports
            python -m unittest opacus.tests.multigpu_gradcheck.GradientComputationTest.test_gradient_correct


  auto_deploy_site:
    docker:
      - image: cimg/python:3.9-node
    steps:
      - run: node --version
      - run: yarn --version
      - checkout
      - pip_dev_install:
          args: "-n -d"
      - configure_docusaurus_bot
      - deploy_site


aliases:

  - &exclude_ghpages
    branches:
      ignore:
        - gh-pages

# -------------------------------------------------------------------------------------
# Workflows
# -------------------------------------------------------------------------------------

workflows:
  commit:
    when:
      not:
        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
    jobs:
      - lint_py37_torch_release:
          filters: *exclude_ghpages
      - unittest_py37_torch_release:
          filters: *exclude_ghpages
      - unittest_py38_torch_release:
          filters: *exclude_ghpages
      - unittest_py39_torch_release:
          filters: *exclude_ghpages
      - unittest_py39_torch_nightly:
          filters: *exclude_ghpages
      - unittest_multi_gpu:
          filters: *exclude_ghpages
      - integrationtest_py37_torch_release_cpu:
          filters: *exclude_ghpages
      - integrationtest_py37_torch_release_cuda:
          filters: *exclude_ghpages

  nightly:
    when:
      equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
    jobs:
      - unittest_py39_torch_nightly:
          filters: *exclude_ghpages
      - integrationtest_py37_torch_release_cpu:
          filters: *exclude_ghpages
      - integrationtest_py37_torch_release_cuda:
          filters: *exclude_ghpages
      - lint_py37_torch_release:
          filters: *exclude_ghpages

  website_deployment:
    when:
      not:
        equal: [ scheduled_pipeline, << pipeline.trigger_source >> ]
    jobs:
      - auto_deploy_site:
          filters:
            branches:
              only:
                - main