From 51663fe96806b4d59c0dd84500ea1767fdb745af Mon Sep 17 00:00:00 2001 From: driazati Date: Mon, 9 May 2022 16:21:48 -0700 Subject: [PATCH 1/4] [ci] Use r5.large nodes for builds and lint This uses `r5.large` for linting and build steps and splits lint into 2 to keep runtime down. This is a subset split off of #11120. Once `task_cpp_unittest.sh` is fixed so it picks up sccache we can enable these smaller nodes there as well. --- Jenkinsfile | 173 +++++++++++++++++++++++++------------ jenkins/Jenkinsfile.j2 | 91 +++++++++---------- jenkins/macros.j2 | 19 ++++ tests/scripts/task_lint.sh | 85 +++++++++++------- 4 files changed, 231 insertions(+), 137 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a541f1688cc2..5d07e9563b0c 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-05T13:07:33.276898 +// Generated at 2022-05-09T16:19:42.885533 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -189,59 +189,126 @@ cancel_previous_build() def lint() { stage('Lint') { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - ci_lint = params.ci_lint_param ?: ci_lint - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_wasm = params.ci_wasm_param ?: ci_wasm - ci_i386 = params.ci_i386_param ?: ci_i386 - ci_qemu = params.ci_qemu_param ?: ci_qemu - ci_arm = params.ci_arm_param ?: ci_arm - ci_hexagon = params.ci_hexagon_param ?: ci_hexagon + parallel( + 'Lint 1 of 2': { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=0'], { + ci_arm = params.ci_arm_param ?: ci_arm + ci_cpu = params.ci_cpu_param ?: ci_cpu + ci_gpu = params.ci_gpu_param ?: ci_gpu + ci_hexagon = params.ci_hexagon_param ?: ci_hexagon + ci_i386 = params.ci_i386_param ?: ci_i386 + ci_lint = params.ci_lint_param ?: ci_lint + ci_qemu = params.ci_qemu_param ?: ci_qemu + ci_wasm = params.ci_wasm_param ?: ci_wasm - sh (script: """ - echo "Docker images being used in this build:" - echo " ci_lint = ${ci_lint}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_wasm = ${ci_wasm}" - echo " ci_i386 = ${ci_i386}" - echo " ci_qemu = ${ci_qemu}" - echo " ci_arm = ${ci_arm}" - echo " ci_hexagon = ${ci_hexagon}" - """, label: 'Docker image names') + sh (script: """ + echo "Docker images being used in this build:" + echo " ci_arm = ${ci_arm}" + echo " ci_cpu = ${ci_cpu}" + echo " ci_gpu = ${ci_gpu}" + echo " ci_hexagon = ${ci_hexagon}" + echo " ci_i386 = ${ci_i386}" + echo " ci_lint = ${ci_lint}" + echo " ci_qemu = ${ci_qemu}" + echo " ci_wasm = ${ci_wasm}" + """, label: 'Docker image names') - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/sanity") { - init_git() - is_docs_only_build = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docs.sh', - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docker.sh', - label: 'Check for any docker changes', - ) - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false + is_docs_only_build = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docs.sh', + label: 'Check for docs only changes', + ) + skip_ci = should_skip_ci(env.CHANGE_ID) + skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) + rebuild_docker_images = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docker.sh', + label: 'Check for any docker changes', + ) + if (skip_ci) { + // Don't rebuild when skipping CI + rebuild_docker_images = false + } + if (rebuild_docker_images) { + // Exit before linting so we can use the newly created Docker images + // to run the lint + return + } + sh ( + script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + label: 'Run lint', + ) + }) } - if (rebuild_docker_images) { - // Exit before linting so we can use the newly created Docker images - // to run the lint - return + } + } + }, + 'Lint 2 of 2': { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=1'], { + ci_arm = params.ci_arm_param ?: ci_arm + ci_cpu = params.ci_cpu_param ?: ci_cpu + ci_gpu = params.ci_gpu_param ?: ci_gpu + ci_hexagon = params.ci_hexagon_param ?: ci_hexagon + ci_i386 = params.ci_i386_param ?: ci_i386 + ci_lint = params.ci_lint_param ?: ci_lint + ci_qemu = params.ci_qemu_param ?: ci_qemu + ci_wasm = params.ci_wasm_param ?: ci_wasm + + sh (script: """ + echo "Docker images being used in this build:" + echo " ci_arm = ${ci_arm}" + echo " ci_cpu = ${ci_cpu}" + echo " ci_gpu = ${ci_gpu}" + echo " ci_hexagon = ${ci_hexagon}" + echo " ci_i386 = ${ci_i386}" + echo " ci_lint = ${ci_lint}" + echo " ci_qemu = ${ci_qemu}" + echo " ci_wasm = ${ci_wasm}" + """, label: 'Docker image names') + + is_docs_only_build = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docs.sh', + label: 'Check for docs only changes', + ) + skip_ci = should_skip_ci(env.CHANGE_ID) + skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) + rebuild_docker_images = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docker.sh', + label: 'Check for any docker changes', + ) + if (skip_ci) { + // Don't rebuild when skipping CI + rebuild_docker_images = false + } + if (rebuild_docker_images) { + // Exit before linting so we can use the newly created Docker images + // to run the lint + return + } + sh ( + script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + label: 'Run lint', + ) + }) } - sh ( - script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - label: 'Run lint', - ) } } - } + }, + ) } } @@ -499,7 +566,7 @@ stage('Build') { } parallel 'BUILD: GPU': { if (!skip_ci) { - node('CPU') { + node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") { init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" @@ -516,7 +583,7 @@ stage('Build') { }, 'BUILD: CPU': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") { init_git() sh ( @@ -539,7 +606,7 @@ stage('Build') { }, 'BUILD: WASM': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") { init_git() sh ( @@ -563,7 +630,7 @@ stage('Build') { }, 'BUILD: i386': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") { init_git() sh ( @@ -597,7 +664,7 @@ stage('Build') { }, 'BUILD: QEMU': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") { init_git() sh ( diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2 index b961b6146801..a9f8f7accb74 100644 --- a/jenkins/Jenkinsfile.j2 +++ b/jenkins/Jenkinsfile.j2 @@ -186,59 +186,46 @@ cancel_previous_build() def lint() { stage('Lint') { - node('CPU') { - timeout(time: max_time, unit: 'MINUTES') { - ci_lint = params.ci_lint_param ?: ci_lint - ci_cpu = params.ci_cpu_param ?: ci_cpu - ci_gpu = params.ci_gpu_param ?: ci_gpu - ci_wasm = params.ci_wasm_param ?: ci_wasm - ci_i386 = params.ci_i386_param ?: ci_i386 - ci_qemu = params.ci_qemu_param ?: ci_qemu - ci_arm = params.ci_arm_param ?: ci_arm - ci_hexagon = params.ci_hexagon_param ?: ci_hexagon + parallel( + {% call m.sharded_lint_step(name='Lint', num_shards=2, node='CPU-SMALL', ws='tvm/lint') %} + {% for image in images %} + {{ image.name }} = params.{{ image.name }}_param ?: {{ image.name }} + {% endfor %} sh (script: """ echo "Docker images being used in this build:" - echo " ci_lint = ${ci_lint}" - echo " ci_cpu = ${ci_cpu}" - echo " ci_gpu = ${ci_gpu}" - echo " ci_wasm = ${ci_wasm}" - echo " ci_i386 = ${ci_i386}" - echo " ci_qemu = ${ci_qemu}" - echo " ci_arm = ${ci_arm}" - echo " ci_hexagon = ${ci_hexagon}" + {% for image in images %} + echo " {{ image.name }} = ${ {{- image.name -}} }" + {% endfor %} """, label: 'Docker image names') - ws({{ m.per_exec_ws('tvm/sanity') }}) { - init_git() - is_docs_only_build = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docs.sh', - label: 'Check for docs only changes', - ) - skip_ci = should_skip_ci(env.CHANGE_ID) - skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) - rebuild_docker_images = sh ( - returnStatus: true, - script: './tests/scripts/git_change_docker.sh', - label: 'Check for any docker changes', - ) - if (skip_ci) { - // Don't rebuild when skipping CI - rebuild_docker_images = false - } - if (rebuild_docker_images) { - // Exit before linting so we can use the newly created Docker images - // to run the lint - return - } - sh ( - script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", - label: 'Run lint', - ) + is_docs_only_build = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docs.sh', + label: 'Check for docs only changes', + ) + skip_ci = should_skip_ci(env.CHANGE_ID) + skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) + rebuild_docker_images = sh ( + returnStatus: true, + script: './tests/scripts/git_change_docker.sh', + label: 'Check for any docker changes', + ) + if (skip_ci) { + // Don't rebuild when skipping CI + rebuild_docker_images = false } - } - } + if (rebuild_docker_images) { + // Exit before linting so we can use the newly created Docker images + // to run the lint + return + } + sh ( + script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh", + label: 'Run lint', + ) + {% endcall %} + ) } } @@ -496,7 +483,7 @@ stage('Build') { } parallel 'BUILD: GPU': { if (!skip_ci) { - node('CPU') { + node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-gpu') }}) { init_git() sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build" @@ -513,7 +500,7 @@ stage('Build') { }, 'BUILD: CPU': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-cpu') }}) { init_git() sh ( @@ -536,7 +523,7 @@ stage('Build') { }, 'BUILD: WASM': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-wasm') }}) { init_git() sh ( @@ -560,7 +547,7 @@ stage('Build') { }, 'BUILD: i386': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-i386') }}) { init_git() sh ( @@ -594,7 +581,7 @@ stage('Build') { }, 'BUILD: QEMU': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws({{ m.per_exec_ws('tvm/build-qemu') }}) { init_git() sh ( diff --git a/jenkins/macros.j2 b/jenkins/macros.j2 index 97e6eee68c75..cbb36545ff5b 100644 --- a/jenkins/macros.j2 +++ b/jenkins/macros.j2 @@ -47,6 +47,25 @@ {% endfor %} {% endmacro %} +{% macro sharded_lint_step(name, num_shards, node, ws) %} +{% for shard_index in range(1, num_shards + 1) %} + '{{ name }} {{ shard_index }} of {{ num_shards }}': { + node('{{ node }}') { + ws({{ per_exec_ws(ws) }}) { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'TVM_NUM_SHARDS={{ num_shards }}', + 'TVM_SHARD_INDEX={{ shard_index - 1 }}'], { + {{ caller() | trim | indent(width=6) }} + }) + } + } + } + }, +{% endfor %} +{% endmacro %} + {% macro test_step(name, node, ws, platform) %} '{{ name }}': { diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh index 79e86f0c49e6..ea67da673f87 100755 --- a/tests/scripts/task_lint.sh +++ b/tests/scripts/task_lint.sh @@ -25,48 +25,69 @@ cleanup() trap cleanup 0 -echo "Convert scripts to Python..." -tests/scripts/task_convert_scripts_to_python.sh +# These shards are solely for CI to enable the lint job to have some parallelism. -# TODO: Remove this ad-hoc pip install once https://github.com/apache/tvm/pull/10741 -# is added to the ci_lint Docker image -python3 -m pip install --user -r jenkins/requirements.txt -echo "Check Jenkinsfile generation" -python3 jenkins/generate.py --check +function shard1 { + echo "Convert scripts to Python..." + tests/scripts/task_convert_scripts_to_python.sh -echo "Checking file types..." -python3 tests/lint/check_file_type.py + # TODO: Remove this ad-hoc pip install once https://github.com/apache/tvm/pull/10741 + # is added to the ci_lint Docker image + python3 -m pip install --user -r jenkins/requirements.txt + echo "Check Jenkinsfile generation" + python3 jenkins/generate.py --check -echo "Checking CMake <-> LibInfo options mirroring" -python3 tests/lint/check_cmake_options.py + echo "Checking file types..." + python3 tests/lint/check_file_type.py -echo "Checking ASF license headers..." -tests/lint/check_asf_header.sh --local + echo "Checking CMake <-> LibInfo options mirroring" + python3 tests/lint/check_cmake_options.py -echo "Linting the C++ code..." -tests/lint/cpplint.sh + echo "black check..." + tests/lint/git-black.sh -echo "clang-format check..." -tests/lint/git-clang-format.sh + echo "Linting the Python code with flake8..." + tests/lint/flake8.sh -echo "Rust check..." -tests/lint/rust_format.sh + echo "Type checking with MyPy ..." + tests/scripts/task_mypy.sh -echo "black check..." -tests/lint/git-black.sh + echo "Checking for non-inclusive language with blocklint..." + tests/lint/blocklint.sh -echo "Linting the Python code..." -tests/lint/pylint.sh -tests/lint/flake8.sh + echo "Linting the JNI code..." + tests/lint/jnilint.sh +} + +function shard2 { + echo "Linting the Python code with pylint..." + tests/lint/pylint.sh + + echo "Checking C++ documentation..." + tests/lint/cppdocs.sh + + echo "Checking ASF license headers..." + tests/lint/check_asf_header.sh --local -echo "Linting the JNI code..." -tests/lint/jnilint.sh + echo "Linting the C++ code..." + tests/lint/cpplint.sh + + echo "clang-format check..." + tests/lint/git-clang-format.sh + + echo "Rust check..." + tests/lint/rust_format.sh +} -echo "Checking C++ documentation..." -tests/lint/cppdocs.sh -echo "Type checking with MyPy ..." -tests/scripts/task_mypy.sh +if [[ -n ${TVM_SHARD_INDEX+x} ]]; then + if [[ "$TVM_SHARD_INDEX" == "0" ]]; then + shard1 + else + shard2 + fi +else + shard1 + shard2 +fi -echo "Checking for non-inclusive language with blocklint..." -tests/lint/blocklint.sh \ No newline at end of file From c06042f8299a21998632fb6642951757ae5c7c3c Mon Sep 17 00:00:00 2001 From: driazati Date: Tue, 10 May 2022 10:53:13 -0700 Subject: [PATCH 2/4] Remove useless pip install --- tests/scripts/task_lint.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh index ea67da673f87..8fbba52662de 100755 --- a/tests/scripts/task_lint.sh +++ b/tests/scripts/task_lint.sh @@ -31,9 +31,6 @@ function shard1 { echo "Convert scripts to Python..." tests/scripts/task_convert_scripts_to_python.sh - # TODO: Remove this ad-hoc pip install once https://github.com/apache/tvm/pull/10741 - # is added to the ci_lint Docker image - python3 -m pip install --user -r jenkins/requirements.txt echo "Check Jenkinsfile generation" python3 jenkins/generate.py --check From 95a885932aa062ad804b5efd3eb2a37056ec78fb Mon Sep 17 00:00:00 2001 From: driazati Date: Tue, 10 May 2022 12:05:22 -0700 Subject: [PATCH 3/4] Revert "Remove useless pip install" This reverts commit e06cdcfbf377d35e3f6ee191f7ee8ebada699a84. --- tests/scripts/task_lint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh index 8fbba52662de..ea67da673f87 100755 --- a/tests/scripts/task_lint.sh +++ b/tests/scripts/task_lint.sh @@ -31,6 +31,9 @@ function shard1 { echo "Convert scripts to Python..." tests/scripts/task_convert_scripts_to_python.sh + # TODO: Remove this ad-hoc pip install once https://github.com/apache/tvm/pull/10741 + # is added to the ci_lint Docker image + python3 -m pip install --user -r jenkins/requirements.txt echo "Check Jenkinsfile generation" python3 jenkins/generate.py --check From 64f00e623be594976f98f85a894571cc6c67dfe7 Mon Sep 17 00:00:00 2001 From: driazati Date: Tue, 10 May 2022 12:08:21 -0700 Subject: [PATCH 4/4] Update lint --- tests/scripts/task_lint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/scripts/task_lint.sh b/tests/scripts/task_lint.sh index ea67da673f87..e0c953d61841 100755 --- a/tests/scripts/task_lint.sh +++ b/tests/scripts/task_lint.sh @@ -31,7 +31,7 @@ function shard1 { echo "Convert scripts to Python..." tests/scripts/task_convert_scripts_to_python.sh - # TODO: Remove this ad-hoc pip install once https://github.com/apache/tvm/pull/10741 + # TODO: Remove this ad-hoc pip install once https://github.com/apache/tvm/pull/11265 # is added to the ci_lint Docker image python3 -m pip install --user -r jenkins/requirements.txt echo "Check Jenkinsfile generation"