Skip to content
Permalink
master
Switch branches/tags

Name already in use

A tag already exists with the provided branch name. Many Git commands accept both tag and branch names, so creating this branch may cause unexpected behavior. Are you sure you want to create this branch?
Go to file
 
 
Cannot retrieve contributors at this time
# Copyright (c) 2020 The Neuropod Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This script lets us define a CI build matrix in one place
import itertools
# Template for .github/workflows/mac_ci.yml
GH_ACTIONS_TEMPLATE = """
#
# DO NOT MANUALLY EDIT THIS FILE
# AUTOGENERATED BY build/ci_matrix.py
#
name: Mac CI
on:
push:
branches:
- master
tags:
- v*
pull_request:
jobs:
build:
runs-on: macos-10.15
name: TF ${{{{ matrix.tf }}}}, Torch ${{{{ matrix.torch }}}}, Python ${{{{ matrix.python }}}}
env:
NEUROPOD_TENSORFLOW_VERSION: ${{{{ matrix.tf }}}}
NEUROPOD_TORCH_VERSION: ${{{{ matrix.torch }}}}
NEUROPOD_PYTHON_VERSION: ${{{{ matrix.python }}}}
steps:
# Checkout the repo
- uses: actions/checkout@v2
# Setup bazel cache
- name: Bazel Cache
uses: actions/cache@v2
with:
path: ~/bazel_cache
key: ${{{{ runner.os }}}}-bazel-cache-tf-${{{{ matrix.tf }}}}-torch-${{{{ matrix.torch }}}}-python-${{{{ matrix.python }}}}
# Build and test
- name: Build and Test
run: ./build/ci/gh_actions_build.sh
env:
NEUROPOD_TEST_FRAMEWORKS: ${{{{ matrix.test_frameworks }}}}
GH_UPLOAD_TOKEN: ${{{{ secrets.GITHUB_TOKEN }}}}
strategy:
matrix:
include:
{}
"""
# Template for docker-compose.test.yml
DOCKER_COMPOSE_TEST_TEMPLATE = """
#
# DO NOT MANUALLY EDIT THIS FILE
# AUTOGENERATED BY build/ci_matrix.py
#
version: '2.3'
services:
test-base:
build:
context: .
dockerfile: build/neuropod.dockerfile
target: neuropod-base
test-gpu:
extends: test-base
build:
args:
NEUROPOD_IS_GPU: "true"
runtime: nvidia
{}
"""
# Template for .buildkite/pipeline.yml
BUILDKITE_YML_TEMPLATE = """
#
# DO NOT MANUALLY EDIT THIS FILE
# AUTOGENERATED BY build/ci_matrix.py
#
steps:
{}
"""
# The platforms we're testing on
PLATFORMS = [
"linux_cpu",
"linux_gpu",
"macos_cpu",
]
# Versions of frameworks to test with
FRAMEWORK_VERSIONS = [
{"cuda": "10.0", "tensorflow": "1.14.0", "torch": "1.3.0", "python": "3.6"},
{"cuda": "10.0", "tensorflow": "1.15.0", "torch": "1.4.0", "python": "3.7"},
{"cuda": "10.1", "tensorflow": "2.2.0", "torch": "1.5.0", "python": "3.8"},
# No need to rerun tensorflow tests for 2.2.0 on py3.8
{"cuda": "10.1", "tensorflow": "2.2.0", "torch": "1.6.0", "python": "3.8", "test_frameworks": "torchscript,python"},
{"cuda": "10.1", "tensorflow": "2.2.0", "torch": "1.7.0", "python": "3.8", "test_frameworks": "torchscript,python"},
{"cuda": "10.2", "tensorflow": "2.2.0", "torch": "1.8.1", "python": "3.8", "test_frameworks": "torchscript,python"},
{"cuda": "10.2", "tensorflow": "2.2.0", "torch": "1.9.0", "python": "3.8", "test_frameworks": "torchscript,python"},
{"cuda": "10.2", "tensorflow": "2.2.0", "torch": "1.10.2", "python": "3.8", "test_frameworks": "torchscript,python"},
# Only testing TF
{"cuda": "11.2.1", "cudnn": "8", "tensorflow": "2.5.0", "torch": "1.7.0", "python": "3.8", "test_frameworks": "tensorflow"},
{"cuda": "11.2.1", "cudnn": "8", "tensorflow": "2.6.2", "torch": "1.7.0", "python": "3.8", "test_frameworks": "tensorflow"},
]
gh_actions_matrix = []
docker_compose_matrix = []
buildkite_yml_matrix = []
added_lint = False
for platform, framework_version in itertools.product(PLATFORMS, FRAMEWORK_VERSIONS):
# Get versions of all the dependencies
tf_version = framework_version["tensorflow"]
torch_version = framework_version["torch"]
py_version = framework_version["python"]
# Which frameworks we want to run tests for
# TODO(vip): do this better
test_frameworks = framework_version["test_frameworks"] if "test_frameworks" in framework_version else "tensorflow,torchscript,python"
# Generate the appropriate configuration
if "macos" in platform:
# This is a GH Actions build
gh_actions_matrix.extend([
" - tf: {}\n".format(tf_version),
" torch: {}\n".format(torch_version),
" python: {}\n".format(py_version),
" test_frameworks: {}\n".format(test_frameworks),
"\n",
])
elif "linux" in platform:
is_gpu = "gpu" in platform
variant_name = "test-{}-variant-tf-{}-torch-{}-py{}".format("gpu" if is_gpu else "cpu", tf_version, torch_version, py_version).replace(".", "_")
docker_compose_matrix.extend([
" {}:\n".format(variant_name),
" extends: test-{}\n".format("gpu" if is_gpu else "base"),
" build:\n",
" args:\n",
" NEUROPOD_CUDA_VERSION: {}\n".format(framework_version["cuda"]) if is_gpu else "",
" NEUROPOD_CUDNN_VERSION: {}\n".format(framework_version["cudnn"]) if is_gpu and "cudnn" in framework_version else "",
" NEUROPOD_TENSORFLOW_VERSION: {}\n".format(tf_version),
" NEUROPOD_TORCH_VERSION: {}\n".format(torch_version),
" NEUROPOD_PYTHON_VERSION: {}\n".format(py_version),
"\n",
])
plugin_config = [
" plugins:\n",
" - docker-compose#v3.7.0:\n",
" build: {}\n".format(variant_name),
" config: docker-compose.test.yml\n",
" image-repository: 027047743804.dkr.ecr.us-east-2.amazonaws.com/uber\n",
" cache-from: {}:027047743804.dkr.ecr.us-east-2.amazonaws.com/uber:{}\n".format(variant_name, variant_name),
" push-retries: 5\n",
" - docker-compose#v3.7.0:\n",
" push: {}:027047743804.dkr.ecr.us-east-2.amazonaws.com/uber:{}\n".format(variant_name, variant_name),
" config: docker-compose.test.yml\n",
" - docker-compose#v3.7.0:\n",
" run: {}\n".format(variant_name),
" config: docker-compose.test.yml\n",
" env:\n",
" - NEUROPOD_CACHE_ACCESS_KEY\n",
" - NEUROPOD_CACHE_ACCESS_SECRET\n",
" - BUILDKITE\n",
" - BUILDKITE_BRANCH\n",
" - BUILDKITE_BUILD_NUMBER\n",
" - BUILDKITE_BUILD_URL\n",
" - BUILDKITE_COMMIT\n",
" - BUILDKITE_JOB_ID\n",
" - BUILDKITE_PROJECT_SLUG\n",
" - BUILDKITE_PULL_REQUEST\n",
" - BUILDKITE_TAG\n",
" - CI\n",
" - CODECOV_TOKEN\n",
" - GH_STATUS_TOKEN\n",
" - GH_UPLOAD_TOKEN\n",
" - NEUROPOD_TEST_FRAMEWORKS\n",
" - WEB_DEPLOY_KEY\n",
" retry:\n",
" automatic: true\n",
"\n",
]
buildkite_yml_matrix.extend([
" - label: \":docker: {} Tests ({})\"\n".format("GPU" if is_gpu else "CPU", variant_name),
" timeout_in_minutes: 60\n",
" agents:\n",
" queue: {}\n".format("public-gpu" if is_gpu else "public-gpu"), # Temporarily run everything on `public-gpu`
" env:\n",
" NEUROPOD_TEST_FRAMEWORKS: {}\n".format(test_frameworks),
" command: build/ci/{}.sh\n".format("buildkite_build_gpu" if is_gpu else "buildkite_build"),
] + plugin_config)
if not is_gpu and not added_lint:
# Add a lint job to our build matrix
added_lint = True
buildkite_yml_matrix.extend([
" - label: \":docker: Lint + Docs\"\n".format(variant_name),
" timeout_in_minutes: 60\n",
" agents:\n",
" queue: public-gpu\n", # Temporarily run everything on `public-gpu`
" command: build/ci/buildkite_lint.sh\n",
] + plugin_config)
# Use the templates to create the complete config files
GH_ACTIONS_YML = GH_ACTIONS_TEMPLATE.format("".join(gh_actions_matrix))
DOCKER_COMPOSE_TEST = DOCKER_COMPOSE_TEST_TEMPLATE.format("".join(docker_compose_matrix))
BUILDKITE_YML = BUILDKITE_YML_TEMPLATE.format("".join(buildkite_yml_matrix))
if __name__ == '__main__':
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('--write-config', help=('Actually write all the CI config files (e.g. pipeline.yml, mac_ci.yml, docker-compose.test.yml).'
'Otherwise, just verify that the files match the build matrix defined here'
'Default False.'), default=False, action='store_true')
args = parser.parse_args()
files = {
'./docker-compose.test.yml': DOCKER_COMPOSE_TEST,
'./.github/workflows/mac_ci.yml': GH_ACTIONS_YML,
'./.buildkite/pipeline.yml': BUILDKITE_YML,
}
if args.write_config:
for path, content in files.items():
with open(path, 'w') as f:
f.write(content)
else:
# Just verify that everything matches
for path, target_content in files.items():
with open(path, 'r') as f:
content = f.read()
if content != target_content:
raise ValueError("{} does not match current build matrix! Please run `./build/ci_matrix.py --write-config` to fix.".format(path))