Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[ci] Use r5.large nodes for builds and lint #11258

Merged
merged 4 commits into from
May 11, 2022
Merged

[ci] Use r5.large nodes for builds and lint #11258

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
51663fe
[ci] Use r5.large nodes for builds and lint
driazati May 9, 2022
c06042f
Remove useless pip install
driazati May 10, 2022
95a8859
Revert "Remove useless pip install"
driazati May 10, 2022
64f00e6
Update lint
driazati May 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
173 changes: 120 additions & 53 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
// Generated at 2022-05-05T13:07:33.276898
// Generated at 2022-05-09T16:19:42.885533

import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
Expand Down Expand Up @@ -189,59 +189,126 @@ cancel_previous_build()

def lint() {
stage('Lint') {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
ci_lint = params.ci_lint_param ?: ci_lint
ci_cpu = params.ci_cpu_param ?: ci_cpu
ci_gpu = params.ci_gpu_param ?: ci_gpu
ci_wasm = params.ci_wasm_param ?: ci_wasm
ci_i386 = params.ci_i386_param ?: ci_i386
ci_qemu = params.ci_qemu_param ?: ci_qemu
ci_arm = params.ci_arm_param ?: ci_arm
ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
parallel(
'Lint 1 of 2': {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
'TVM_NUM_SHARDS=2',
'TVM_SHARD_INDEX=0'], {
ci_arm = params.ci_arm_param ?: ci_arm
ci_cpu = params.ci_cpu_param ?: ci_cpu
ci_gpu = params.ci_gpu_param ?: ci_gpu
ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
ci_i386 = params.ci_i386_param ?: ci_i386
ci_lint = params.ci_lint_param ?: ci_lint
ci_qemu = params.ci_qemu_param ?: ci_qemu
ci_wasm = params.ci_wasm_param ?: ci_wasm

sh (script: """
echo "Docker images being used in this build:"
echo " ci_lint = ${ci_lint}"
echo " ci_cpu = ${ci_cpu}"
echo " ci_gpu = ${ci_gpu}"
echo " ci_wasm = ${ci_wasm}"
echo " ci_i386 = ${ci_i386}"
echo " ci_qemu = ${ci_qemu}"
echo " ci_arm = ${ci_arm}"
echo " ci_hexagon = ${ci_hexagon}"
""", label: 'Docker image names')
sh (script: """
echo "Docker images being used in this build:"
echo " ci_arm = ${ci_arm}"
echo " ci_cpu = ${ci_cpu}"
echo " ci_gpu = ${ci_gpu}"
echo " ci_hexagon = ${ci_hexagon}"
echo " ci_i386 = ${ci_i386}"
echo " ci_lint = ${ci_lint}"
echo " ci_qemu = ${ci_qemu}"
echo " ci_wasm = ${ci_wasm}"
""", label: 'Docker image names')

ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/sanity") {
init_git()
is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/git_change_docker.sh',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/git_change_docker.sh',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
}
if (rebuild_docker_images) {
// Exit before linting so we can use the newly created Docker images
// to run the lint
return
}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
})
}
if (rebuild_docker_images) {
// Exit before linting so we can use the newly created Docker images
// to run the lint
return
}
}
},
'Lint 2 of 2': {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/lint") {
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
'TVM_NUM_SHARDS=2',
'TVM_SHARD_INDEX=1'], {
ci_arm = params.ci_arm_param ?: ci_arm
ci_cpu = params.ci_cpu_param ?: ci_cpu
ci_gpu = params.ci_gpu_param ?: ci_gpu
ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
ci_i386 = params.ci_i386_param ?: ci_i386
ci_lint = params.ci_lint_param ?: ci_lint
ci_qemu = params.ci_qemu_param ?: ci_qemu
ci_wasm = params.ci_wasm_param ?: ci_wasm

sh (script: """
echo "Docker images being used in this build:"
echo " ci_arm = ${ci_arm}"
echo " ci_cpu = ${ci_cpu}"
echo " ci_gpu = ${ci_gpu}"
echo " ci_hexagon = ${ci_hexagon}"
echo " ci_i386 = ${ci_i386}"
echo " ci_lint = ${ci_lint}"
echo " ci_qemu = ${ci_qemu}"
echo " ci_wasm = ${ci_wasm}"
""", label: 'Docker image names')

is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/git_change_docker.sh',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
}
if (rebuild_docker_images) {
// Exit before linting so we can use the newly created Docker images
// to run the lint
return
}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
})
}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
}
}
}
},
)
}
}

Expand Down Expand Up @@ -499,7 +566,7 @@ stage('Build') {
}
parallel 'BUILD: GPU': {
if (!skip_ci) {
node('CPU') {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
init_git()
sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
Expand All @@ -516,7 +583,7 @@ stage('Build') {
},
'BUILD: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
init_git()
sh (
Expand All @@ -539,7 +606,7 @@ stage('Build') {
},
'BUILD: WASM': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
init_git()
sh (
Expand All @@ -563,7 +630,7 @@ stage('Build') {
},
'BUILD: i386': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
init_git()
sh (
Expand Down Expand Up @@ -597,7 +664,7 @@ stage('Build') {
},
'BUILD: QEMU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") {
init_git()
sh (
Expand Down
91 changes: 39 additions & 52 deletions jenkins/Jenkinsfile.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,59 +186,46 @@ cancel_previous_build()

def lint() {
stage('Lint') {
node('CPU') {
timeout(time: max_time, unit: 'MINUTES') {
ci_lint = params.ci_lint_param ?: ci_lint
ci_cpu = params.ci_cpu_param ?: ci_cpu
ci_gpu = params.ci_gpu_param ?: ci_gpu
ci_wasm = params.ci_wasm_param ?: ci_wasm
ci_i386 = params.ci_i386_param ?: ci_i386
ci_qemu = params.ci_qemu_param ?: ci_qemu
ci_arm = params.ci_arm_param ?: ci_arm
ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
parallel(
{% call m.sharded_lint_step(name='Lint', num_shards=2, node='CPU-SMALL', ws='tvm/lint') %}
{% for image in images %}
{{ image.name }} = params.{{ image.name }}_param ?: {{ image.name }}
{% endfor %}

sh (script: """
echo "Docker images being used in this build:"
echo " ci_lint = ${ci_lint}"
echo " ci_cpu = ${ci_cpu}"
echo " ci_gpu = ${ci_gpu}"
echo " ci_wasm = ${ci_wasm}"
echo " ci_i386 = ${ci_i386}"
echo " ci_qemu = ${ci_qemu}"
echo " ci_arm = ${ci_arm}"
echo " ci_hexagon = ${ci_hexagon}"
{% for image in images %}
echo " {{ image.name }} = ${ {{- image.name -}} }"
{% endfor %}
""", label: 'Docker image names')

ws({{ m.per_exec_ws('tvm/sanity') }}) {
init_git()
is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/git_change_docker.sh',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
}
if (rebuild_docker_images) {
// Exit before linting so we can use the newly created Docker images
// to run the lint
return
}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
is_docs_only_build = sh (
returnStatus: true,
script: './tests/scripts/git_change_docs.sh',
label: 'Check for docs only changes',
)
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
rebuild_docker_images = sh (
returnStatus: true,
script: './tests/scripts/git_change_docker.sh',
label: 'Check for any docker changes',
)
if (skip_ci) {
// Don't rebuild when skipping CI
rebuild_docker_images = false
}
}
}
if (rebuild_docker_images) {
// Exit before linting so we can use the newly created Docker images
// to run the lint
return
}
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
{% endcall %}
)
}
}

Expand Down Expand Up @@ -496,7 +483,7 @@ stage('Build') {
}
parallel 'BUILD: GPU': {
if (!skip_ci) {
node('CPU') {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-gpu') }}) {
init_git()
sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
Expand All @@ -513,7 +500,7 @@ stage('Build') {
},
'BUILD: CPU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-cpu') }}) {
init_git()
sh (
Expand All @@ -536,7 +523,7 @@ stage('Build') {
},
'BUILD: WASM': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-wasm') }}) {
init_git()
sh (
Expand All @@ -560,7 +547,7 @@ stage('Build') {
},
'BUILD: i386': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-i386') }}) {
init_git()
sh (
Expand Down Expand Up @@ -594,7 +581,7 @@ stage('Build') {
},
'BUILD: QEMU': {
if (!skip_ci && is_docs_only_build != 1) {
node('CPU') {
node('CPU-SMALL') {
ws({{ m.per_exec_ws('tvm/build-qemu') }}) {
init_git()
sh (
Expand Down
19 changes: 19 additions & 0 deletions jenkins/macros.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,25 @@
{% endfor %}
{% endmacro %}

{% macro sharded_lint_step(name, num_shards, node, ws) %}
{% for shard_index in range(1, num_shards + 1) %}
'{{ name }} {{ shard_index }} of {{ num_shards }}': {
node('{{ node }}') {
ws({{ per_exec_ws(ws) }}) {
init_git()
timeout(time: max_time, unit: 'MINUTES') {
withEnv([
'TVM_NUM_SHARDS={{ num_shards }}',
'TVM_SHARD_INDEX={{ shard_index - 1 }}'], {
{{ caller() | trim | indent(width=6) }}
})
}
}
}
},
{% endfor %}
{% endmacro %}


{% macro test_step(name, node, ws, platform) %}
'{{ name }}': {
Expand Down
Loading