ci_matrix.py

# Copyright (c) 2020 The Neuropod Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This script lets us define a CI build matrix in one place
import itertools

# Template for .github/workflows/mac_ci.yml
GH_ACTIONS_TEMPLATE = """
#
# DO NOT MANUALLY EDIT THIS FILE
# AUTOGENERATED BY build/ci_matrix.py
#

name: Mac CI
on:
  push:
    branches:
      - master
    tags:
      - v*
  pull_request:
jobs:
    build:
        runs-on: macos-10.15
        name: TF ${{{{ matrix.tf }}}}, Torch ${{{{ matrix.torch }}}}, Python ${{{{ matrix.python }}}}
        env:
            NEUROPOD_TENSORFLOW_VERSION: ${{{{ matrix.tf }}}}
            NEUROPOD_TORCH_VERSION: ${{{{ matrix.torch }}}}
            NEUROPOD_PYTHON_VERSION: ${{{{ matrix.python }}}}
        steps:
            # Checkout the repo
            - uses: actions/checkout@v2
            # Setup bazel cache
            - name: Bazel Cache
              uses: actions/cache@v2
              with:
                path: ~/bazel_cache
                key: ${{{{ runner.os }}}}-bazel-cache-tf-${{{{ matrix.tf }}}}-torch-${{{{ matrix.torch }}}}-python-${{{{ matrix.python }}}}
            # Build and test
            - name: Build and Test
              run: ./build/ci/gh_actions_build.sh
              env:
                NEUROPOD_TEST_FRAMEWORKS: ${{{{ matrix.test_frameworks }}}}
                GH_UPLOAD_TOKEN: ${{{{ secrets.GITHUB_TOKEN }}}}
        strategy:
            matrix:
                include:
{}
"""

# Template for docker-compose.test.yml
DOCKER_COMPOSE_TEST_TEMPLATE = """
#
# DO NOT MANUALLY EDIT THIS FILE
# AUTOGENERATED BY build/ci_matrix.py
#

version: '2.3'
services:
  test-base:
    build:
      context: .
      dockerfile: build/neuropod.dockerfile
      target: neuropod-base

  test-gpu:
    extends: test-base
    build:
      args:
        NEUROPOD_IS_GPU: "true"
    runtime: nvidia

{}
"""

# Template for .buildkite/pipeline.yml
BUILDKITE_YML_TEMPLATE = """
#
# DO NOT MANUALLY EDIT THIS FILE
# AUTOGENERATED BY build/ci_matrix.py
#

steps:
{}
"""

# The platforms we're testing on
PLATFORMS = [
    "linux_cpu",
    "linux_gpu",
    "macos_cpu",
]

# Versions of frameworks to test with
FRAMEWORK_VERSIONS = [
    {"cuda": "10.0", "tensorflow": "1.14.0", "torch": "1.3.0", "python": "3.6"},
    {"cuda": "10.0", "tensorflow": "1.15.0", "torch": "1.4.0", "python": "3.7"},
    {"cuda": "10.1", "tensorflow": "2.2.0", "torch": "1.5.0", "python": "3.8"},

    # No need to rerun tensorflow tests for 2.2.0 on py3.8
    {"cuda": "10.1", "tensorflow": "2.2.0", "torch": "1.6.0", "python": "3.8", "test_frameworks": "torchscript,python"},
    {"cuda": "10.1", "tensorflow": "2.2.0", "torch": "1.7.0", "python": "3.8", "test_frameworks": "torchscript,python"},
    {"cuda": "10.2", "tensorflow": "2.2.0", "torch": "1.8.1", "python": "3.8", "test_frameworks": "torchscript,python"},
    {"cuda": "10.2", "tensorflow": "2.2.0", "torch": "1.9.0", "python": "3.8", "test_frameworks": "torchscript,python"},
    {"cuda": "10.2", "tensorflow": "2.2.0", "torch": "1.10.2", "python": "3.8", "test_frameworks": "torchscript,python"},

    # Only testing TF
    {"cuda": "11.2.1", "cudnn": "8", "tensorflow": "2.5.0", "torch": "1.7.0", "python": "3.8", "test_frameworks": "tensorflow"},
    {"cuda": "11.2.1", "cudnn": "8", "tensorflow": "2.6.2", "torch": "1.7.0", "python": "3.8", "test_frameworks": "tensorflow"},
]

gh_actions_matrix = []
docker_compose_matrix = []
buildkite_yml_matrix = []
added_lint = False
for platform, framework_version in itertools.product(PLATFORMS, FRAMEWORK_VERSIONS):
    # Get versions of all the dependencies
    tf_version = framework_version["tensorflow"]
    torch_version = framework_version["torch"]
    py_version = framework_version["python"]

    # Which frameworks we want to run tests for
    # TODO(vip): do this better
    test_frameworks = framework_version["test_frameworks"] if "test_frameworks" in framework_version else "tensorflow,torchscript,python"

    # Generate the appropriate configuration
    if "macos" in platform:
        # This is a GH Actions build
        gh_actions_matrix.extend([
        "                    - tf: {}\n".format(tf_version),
        "                      torch: {}\n".format(torch_version),
        "                      python: {}\n".format(py_version),
        "                      test_frameworks: {}\n".format(test_frameworks),
        "\n",
        ])

    elif "linux" in platform:
        is_gpu = "gpu" in platform
        variant_name = "test-{}-variant-tf-{}-torch-{}-py{}".format("gpu" if is_gpu else "cpu", tf_version, torch_version, py_version).replace(".", "_")

        docker_compose_matrix.extend([
        "  {}:\n".format(variant_name),
        "    extends: test-{}\n".format("gpu" if is_gpu else "base"),
        "    build:\n",
        "      args:\n",
        "        NEUROPOD_CUDA_VERSION: {}\n".format(framework_version["cuda"]) if is_gpu else "",
        "        NEUROPOD_CUDNN_VERSION: {}\n".format(framework_version["cudnn"]) if is_gpu and "cudnn" in framework_version else "",
        "        NEUROPOD_TENSORFLOW_VERSION: {}\n".format(tf_version),
        "        NEUROPOD_TORCH_VERSION: {}\n".format(torch_version),
        "        NEUROPOD_PYTHON_VERSION: {}\n".format(py_version),
        "\n",
        ])

        plugin_config = [
        "    plugins:\n",
        "      - docker-compose#v3.7.0:\n",
        "          build: {}\n".format(variant_name),
        "          config: docker-compose.test.yml\n",
        "          image-repository: 027047743804.dkr.ecr.us-east-2.amazonaws.com/uber\n",
        "          cache-from: {}:027047743804.dkr.ecr.us-east-2.amazonaws.com/uber:{}\n".format(variant_name, variant_name),
        "          push-retries: 5\n",
        "      - docker-compose#v3.7.0:\n",
        "          push: {}:027047743804.dkr.ecr.us-east-2.amazonaws.com/uber:{}\n".format(variant_name, variant_name),
        "          config: docker-compose.test.yml\n",
        "      - docker-compose#v3.7.0:\n",
        "          run: {}\n".format(variant_name),
        "          config: docker-compose.test.yml\n",
        "          env:\n",
        "            - NEUROPOD_CACHE_ACCESS_KEY\n",
        "            - NEUROPOD_CACHE_ACCESS_SECRET\n",
        "            - BUILDKITE\n",
        "            - BUILDKITE_BRANCH\n",
        "            - BUILDKITE_BUILD_NUMBER\n",
        "            - BUILDKITE_BUILD_URL\n",
        "            - BUILDKITE_COMMIT\n",
        "            - BUILDKITE_JOB_ID\n",
        "            - BUILDKITE_PROJECT_SLUG\n",
        "            - BUILDKITE_PULL_REQUEST\n",
        "            - BUILDKITE_TAG\n",
        "            - CI\n",
        "            - CODECOV_TOKEN\n",
        "            - GH_STATUS_TOKEN\n",
        "            - GH_UPLOAD_TOKEN\n",
        "            - NEUROPOD_TEST_FRAMEWORKS\n",
        "            - WEB_DEPLOY_KEY\n",
        "    retry:\n",
        "      automatic: true\n",
        "\n",
        ]

        buildkite_yml_matrix.extend([
        "  - label: \":docker: {} Tests ({})\"\n".format("GPU" if is_gpu else "CPU", variant_name),
        "    timeout_in_minutes: 60\n",
        "    agents:\n",
        "      queue: {}\n".format("public-gpu" if is_gpu else "public-gpu"), # Temporarily run everything on `public-gpu`
        "    env:\n",
        "      NEUROPOD_TEST_FRAMEWORKS: {}\n".format(test_frameworks),
        "    command: build/ci/{}.sh\n".format("buildkite_build_gpu" if is_gpu else "buildkite_build"),
        ] + plugin_config)

        if not is_gpu and not added_lint:
            # Add a lint job to our build matrix
            added_lint = True

            buildkite_yml_matrix.extend([
            "  - label: \":docker: Lint + Docs\"\n".format(variant_name),
            "    timeout_in_minutes: 60\n",
            "    agents:\n",
            "      queue: public-gpu\n",  # Temporarily run everything on `public-gpu`
            "    command: build/ci/buildkite_lint.sh\n",
            ] + plugin_config)


# Use the templates to create the complete config files
GH_ACTIONS_YML = GH_ACTIONS_TEMPLATE.format("".join(gh_actions_matrix))
DOCKER_COMPOSE_TEST = DOCKER_COMPOSE_TEST_TEMPLATE.format("".join(docker_compose_matrix))
BUILDKITE_YML = BUILDKITE_YML_TEMPLATE.format("".join(buildkite_yml_matrix))

if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--write-config', help=('Actually write all the CI config files (e.g. pipeline.yml, mac_ci.yml, docker-compose.test.yml).'
                                              'Otherwise, just verify that the files match the build matrix defined here'
                                              'Default False.'), default=False, action='store_true')
    args = parser.parse_args()

    files = {
        './docker-compose.test.yml': DOCKER_COMPOSE_TEST,
        './.github/workflows/mac_ci.yml': GH_ACTIONS_YML,
        './.buildkite/pipeline.yml': BUILDKITE_YML,
    }

    if args.write_config:
        for path, content in files.items():
            with open(path, 'w') as f:
                f.write(content)
    else:
        # Just verify that everything matches
        for path, target_content in files.items():
            with open(path, 'r') as f:
                content = f.read()
                if content != target_content:
                    raise ValueError("{} does not match current build matrix! Please run `./build/ci_matrix.py --write-config` to fix.".format(path))