From 834517157264f90abd6466a041b6dd912cb10ad1 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 28 Sep 2017 15:34:23 -0700 Subject: [PATCH 1/5] Allow users to skip Java or Python branch via a comment. Also use try/catch blocks to stop non-blocking branches from stopping the build. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 9abf39d46ece..85af1729f87a 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -36,6 +36,19 @@ List commitArg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/ int javaBuildNum = NO_BUILD +boolean testJava = true +boolean testPython = true + +String commentLower = ghprbCommentBody.toLowerCase() + +if (!commentLower.isEmpty()) { + if (commentLower.endsWith('python only')) { + testJava = false + } else if (commentLower.endsWith('java only')) { + testPython = false + } +} + // This (and the below) define "Stages" of a pipeline. These stages run serially, and inside can // have "parallel" blocks which execute several work steps concurrently. This work is limited to // simple operations -- more complicated operations need to be performed on an actual node. In this @@ -43,13 +56,25 @@ int javaBuildNum = NO_BUILD stage('Build') { parallel ( java: { - def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs - if(javaBuild.getResult() == Result.SUCCESS.toString()) { - javaBuildNum = javaBuild.getNumber() + if (testJava) { + def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs + if (javaBuild.getResult() == Result.SUCCESS.toString()) { + javaBuildNum = javaBuild.getNumber() + } + } else { + echo 'Skipping Java due to comment ending in "python only": ' + ghprbCommentBody } }, python_unit: { // Python doesn't have a build phase, so we include this here. - build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs + if (testPython) { + try { + build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs + } catch (Exception e) { + echo 'Python build failed: ' + e.toString() + } + } else { + echo 'Skipping Python due to comment ending in "java only": ' + ghprbCommentBody + } } ) } @@ -70,7 +95,11 @@ stage('Unit Test / Code Health') { }, java_codehealth: { if(javaBuildNum != NO_BUILD) { - build job: 'beam_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs + try { + build job: 'beam_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs + } catch (Exception e) { + echo 'Java CodeHealth Build Failed: ' + e.toString() + } } } ) From 5cdae8872724b411e3e5b2e7c1e723279eac117a Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 28 Sep 2017 15:55:11 -0700 Subject: [PATCH 2/5] Modify trigger phrase to require specifying which language. Signed-off-by: Jason Kuster --- .test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy index 2e1ea55bfe94..3e7c8c93fd2b 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -58,7 +58,7 @@ pipelineJob('beam_PreCommit_Pipeline') { allowMembersOfWhitelistedOrgsAsAdmin() permitAll() // Remove once Pipeline Build is default. - triggerPhrase('Run PreCommit Pipeline') + triggerPhrase('^Run PreCommit Pipeline (((Python|Java) Only)|All)$') onlyTriggerPhrase() displayBuildErrorsOnDownstreamBuilds() extensions { From 9625de53ee1806f9dce8a896c14132066ee51d41 Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 28 Sep 2017 18:17:07 -0700 Subject: [PATCH 3/5] Introduce a more sustainable way to add new SDKs. Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 31 +++++++++++++------ .../job_beam_PreCommit_Pipeline.groovy | 2 +- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 85af1729f87a..0f104d946134 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -36,16 +36,27 @@ List commitArg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/ int javaBuildNum = NO_BUILD -boolean testJava = true -boolean testPython = true +final String JAVA_BUILD_TYPE = "java" +final String PYTHON_BUILD_TYPE = "python" +final String ALL_BUILD_TYPE = "all" +def buildTypes = [ + JAVA_BUILD_TYPE, + PYTHON_BUILD_TYPE, + ALL_BUILD_TYPE, +] + +String currentBuildType = allBuildType String commentLower = ghprbCommentBody.toLowerCase() +// Currently if there is nothing selected (e.g. the comment is just "retest this please") we select "all" by default. +// In the future we should provide some mechanism, either via commenting or the suite failure message, to enforce +// selection of one of the build types. if (!commentLower.isEmpty()) { - if (commentLower.endsWith('python only')) { - testJava = false - } else if (commentLower.endsWith('java only')) { - testPython = false + commentSplit = commentLower.split(' ') + buildType = commentSplit[commentSplit.length-1] + if (buildTypes.contains(buildType)) { + currentBuildType = buildType } } @@ -56,24 +67,24 @@ if (!commentLower.isEmpty()) { stage('Build') { parallel ( java: { - if (testJava) { + if (currentBuildType == javaBuildType || currentBuildType == allBuildType) { def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs if (javaBuild.getResult() == Result.SUCCESS.toString()) { javaBuildNum = javaBuild.getNumber() } } else { - echo 'Skipping Java due to comment ending in "python only": ' + ghprbCommentBody + echo 'Skipping Java due to comment selecting non-Java execution: ' + ghprbCommentBody } }, python_unit: { // Python doesn't have a build phase, so we include this here. - if (testPython) { + if (currentBuildType == pythonBuildType || currentBuildType == allBuildType) { try { build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs } catch (Exception e) { echo 'Python build failed: ' + e.toString() } } else { - echo 'Skipping Python due to comment ending in "java only": ' + ghprbCommentBody + echo 'Skipping Python due to comment selecting non-Python execution: ' + ghprbCommentBody } } ) diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy index 3e7c8c93fd2b..dadc10cd763a 100644 --- a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy @@ -58,7 +58,7 @@ pipelineJob('beam_PreCommit_Pipeline') { allowMembersOfWhitelistedOrgsAsAdmin() permitAll() // Remove once Pipeline Build is default. - triggerPhrase('^Run PreCommit Pipeline (((Python|Java) Only)|All)$') + triggerPhrase('^Run PreCommit Pipeline (((Python|Java))|All)$') onlyTriggerPhrase() displayBuildErrorsOnDownstreamBuilds() extensions { From daceeea5cd32a9f246a3b95961aa89002725f17c Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 28 Sep 2017 19:25:53 -0700 Subject: [PATCH 4/5] test changing some python things to make build faster Signed-off-by: Jason Kuster --- sdks/python/gen_protos.py | 6 +++++- sdks/python/pom.xml | 24 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/sdks/python/gen_protos.py b/sdks/python/gen_protos.py index d70158bf6a94..ba9be7071e77 100644 --- a/sdks/python/gen_protos.py +++ b/sdks/python/gen_protos.py @@ -21,7 +21,9 @@ import logging import multiprocessing import os +import pip import pkg_resources +import pprint import shutil import subprocess import sys @@ -117,8 +119,10 @@ def _install_grpcio_tools_and_generate_proto_files(): logging.warning('Installing grpcio-tools into %s' % install_path) try: start = time.time() + pprint.pprint(pip.pep425tags.get_supported()) subprocess.check_call( - ['pip', 'install', '--target', install_path, '--build', build_path, + [sys.executable, '-m', 'pip', 'install', + '--target', install_path, '--build', build_path, '--upgrade', GRPC_TOOLS]) logging.warning( 'Installing grpcio-tools took %0.2f seconds.' % (time.time() - start)) diff --git a/sdks/python/pom.xml b/sdks/python/pom.xml index 37f95290a074..c687c8618dcf 100644 --- a/sdks/python/pom.xml +++ b/sdks/python/pom.xml @@ -100,6 +100,26 @@ + + setup-compile-grpcio + compile + + exec + + + ${python.pip.bin} + + install + --user + --upgrade + --ignore-installed + grpcio-tools + + + ${python.user.base} + + + setuptools-build compile @@ -143,7 +163,7 @@ --user --upgrade --ignore-installed - detox + tox ${python.user.base} @@ -197,7 +217,7 @@ exec - ${python.user.base}/bin/detox + ${python.user.base}/bin/tox -e ALL From debc7cc92dc6c06112e15739727bc09610b7d2fb Mon Sep 17 00:00:00 2001 From: Jason Kuster Date: Thu, 28 Sep 2017 19:42:40 -0700 Subject: [PATCH 5/5] pipeline fix Signed-off-by: Jason Kuster --- .test-infra/jenkins/PreCommit_Pipeline.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy index 0f104d946134..131c79845ab5 100644 --- a/.test-infra/jenkins/PreCommit_Pipeline.groovy +++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy @@ -46,7 +46,7 @@ def buildTypes = [ ALL_BUILD_TYPE, ] -String currentBuildType = allBuildType +String currentBuildType = ALL_BUILD_TYPE String commentLower = ghprbCommentBody.toLowerCase() // Currently if there is nothing selected (e.g. the comment is just "retest this please") we select "all" by default. @@ -67,7 +67,7 @@ if (!commentLower.isEmpty()) { stage('Build') { parallel ( java: { - if (currentBuildType == javaBuildType || currentBuildType == allBuildType) { + if (currentBuildType == JAVA_BUILD_TYPE || currentBuildType == ALL_BUILD_TYPE) { def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs if (javaBuild.getResult() == Result.SUCCESS.toString()) { javaBuildNum = javaBuild.getNumber() @@ -77,7 +77,7 @@ stage('Build') { } }, python_unit: { // Python doesn't have a build phase, so we include this here. - if (currentBuildType == pythonBuildType || currentBuildType == allBuildType) { + if (currentBuildType == PYTHON_BUILD_TYPE || currentBuildType == ALL_BUILD_TYPE) { try { build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs } catch (Exception e) {