From 6e71e4de1039466dd30c076ea57b4d7faf57d755 Mon Sep 17 00:00:00 2001 From: iamxy Date: Thu, 11 May 2017 22:38:03 +0800 Subject: [PATCH 1/2] Jenkinsfile: init adding Jenkinsfile --- Jenkinsfile | 252 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100644 Jenkinsfile diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000000..8afa5e9a5fcf --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,252 @@ +#!groovy + +node { + def TIDB_TEST_BRANCH = "master" + def TIDB_BRANCH = "master" + def TIKV_BRANCH = "master" + def UCLOUD_OSS_URL = "http://pingcap-dev.hk.ufileos.com" + + env.GOROOT = "/usr/local/go" + env.GOPATH = "/go" + env.PATH = "${env.GOROOT}/bin:/home/jenkins/bin:/bin:${env.PATH}" + + catchError { + stage('Prepare') { + // pd + node('centos7_build') { + def ws = pwd() + dir("go/src/github.com/pingcap/pd") { + // checkout + checkout scm + // build + sh "GOPATH=${ws}/go:$GOPATH make" + } + stash includes: "go/src/github.com/pingcap/pd/**", name: "pd" + } + + // tidb + node('centos7_build') { + def ws = pwd() + dir("go/src/github.com/pingcap/tidb") { + // checkout + git changelog: false, credentialsId: 'github-iamxy-ssh', poll: false, url: 'git@github.com:pingcap/tidb.git', branch: "${TIDB_BRANCH}" + sh "GOPATH=${ws}/go:$GOPATH make parser" + def tidb_sha1 = sh(returnStdout: true, script: "curl ${UCLOUD_OSS_URL}/refs/pingcap/tidb/${TIDB_BRANCH}/centos7/sha1").trim() + sh "curl ${UCLOUD_OSS_URL}/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz | tar xz" + } + stash includes: "go/src/github.com/pingcap/tidb/**", name: "tidb" + } + + // tidb-test + dir("go/src/github.com/pingcap/tidb-test") { + // checkout + git changelog: false, credentialsId: 'github-iamxy-ssh', poll: false, url: 'git@github.com:pingcap/tidb-test.git', branch: "${TIDB_TEST_BRANCH}" + } + stash includes: "go/src/github.com/pingcap/tidb-test/**", name: "tidb-test" + + // tikv + def tikv_sha1 = sh(returnStdout: true, script: "curl ${UCLOUD_OSS_URL}/refs/pingcap/tikv/${TIKV_BRANCH}/centos7/sha1").trim() + sh "curl ${UCLOUD_OSS_URL}/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz" + + unstash 'pd' + sh "cp go/src/github.com/pingcap/pd/bin/pd-server bin/pd-server && rm -rf go/src/github.com/pingcap/pd" + + stash includes: "bin/**", name: "binaries" + } + + stage('Test') { + def tests = [:] + + tests["PD Test"] = { + node("test") { + def ws = pwd() + deleteDir() + unstash 'pd' + + dir("go/src/github.com/pingcap/pd") { + sh "GOPATH=${ws}/go:$GOPATH make test" + } + } + } + + def run_integration_ddl_test = { ddltest -> + def ws = pwd() + deleteDir() + unstash 'tidb' + unstash 'tidb-test' + unstash 'binaries' + + try { + sh """ + killall -9 ddltest_tidb-server || true + killall -9 tikv-server || true + killall -9 pd-server || true + bin/pd-server --name=pd --data-dir=pd &>pd_ddl_test.log & + sleep 10 + bin/tikv-server --pd-endpoints=127.0.0.1:2379 --data-dir=tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 &>tikv_ddl_test.log & + sleep 10 + """ + + timeout(10) { + dir("go/src/github.com/pingcap/tidb-test") { + sh """ + ln -s tidb/_vendor/src ../vendor + cp ${ws}/go/src/github.com/pingcap/tidb/bin/tidb-server ddl_test/ddltest_tidb-server + cd ddl_test && GOPATH=${ws}/go:$GOPATH ./run-tests.sh -check.f='${ddltest}' + """ + } + } + } catch (err) { + throw err + } finally { + sh "killall -9 ddltest_tidb-server || true" + sh "killall -9 tikv-server || true" + sh "killall -9 pd-server || true" + } + } + + tests["Integration DDL Insert Test"] = { + node("test") { + run_integration_ddl_test('TestDDLSuite.TestSimple.*Insert') + } + } + + tests["Integration DDL Update Test"] = { + node("test") { + run_integration_ddl_test('TestDDLSuite.TestSimple.*Update') + } + } + + tests["Integration DDL Delete Test"] = { + node("test") { + run_integration_ddl_test('TestDDLSuite.TestSimple.*Delete') + } + } + + tests["Integration DDL Other Test"] = { + node("test") { + run_integration_ddl_test('TestDDLSuite.TestSimp(le\$|leMixed|leInc)') + } + } + + tests["Integration DDL Column and Index Test"] = { + node("test") { + run_integration_ddl_test('TestDDLSuite.Test(Column|Index)') + } + } + + tests["Integration Connection Test"] = { + node("test") { + def ws = pwd() + deleteDir() + unstash 'tidb' + unstash 'tidb-test' + unstash 'binaries' + + try { + sh """ + killall -9 tikv-server || true + killall -9 pd-server || true + bin/pd-server --name=pd --data-dir=pd &>pd_conntest.log & + sleep 10 + bin/tikv-server --pd-endpoints=127.0.0.1:2379 --data-dir=tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 &>tikv_conntest.log & + sleep 10 + """ + + dir("go/src/github.com/pingcap/tidb") { + sh """ + GOPATH=`pwd`/_vendor:${ws}/go:$GOPATH CGO_ENABLED=1 go test --args with-tikv store/tikv/*.go + """ + } + } catch (err) { + throw err + } finally { + sh "killall -9 tikv-server || true" + sh "killall -9 pd-server || true" + } + } + } + + def run_integration_other_test = { mytest -> + def ws = pwd() + deleteDir() + unstash 'tidb' + unstash 'tidb-test' + unstash 'binaries' + + try { + sh """ + killall -9 tikv-server || true + killall -9 pd-server || true + bin/pd-server --name=pd --data-dir=pd &>pd_${mytest}.log & + sleep 10 + bin/tikv-server --pd-endpoints=127.0.0.1:2379 --data-dir=tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 &>tikv_${mytest}.log & + sleep 10 + """ + + dir("go/src/github.com/pingcap/tidb-test") { + sh """ + ln -s tidb/_vendor/src ../vendor + GOPATH=${ws}/go:$GOPATH TIKV_PATH='127.0.0.1:2379' TIDB_TEST_STORE_NAME=tikv make ${mytest} + """ + } + } catch (err) { + throw err + } finally { + sh "killall -9 tikv-server || true" + sh "killall -9 pd-server || true" + } + } + + tests["Integration TiDB Test"] = { + node('test') { + run_integration_other_test('tidbtest') + } + } + + tests["Integration MySQL Test"] = { + node("test") { + run_integration_other_test('mysqltest') + } + } + + tests["Integration GORM Test"] = { + node("test") { + run_integration_other_test('gormtest') + } + } + + tests["Integration Go SQL Test"] = { + node("test") { + run_integration_other_test('gosqltest') + } + } + + parallel tests + } + + currentBuild.result = "SUCCESS" + } + + stage('Summary') { + def getChangeLogText = { + def changeLogText = "" + for (int i = 0; i < currentBuild.changeSets.size(); i++) { + for (int j = 0; j < currentBuild.changeSets[i].items.length; j++) { + def commitId = "${currentBuild.changeSets[i].items[j].commitId}" + def commitMsg = "${currentBuild.changeSets[i].items[j].msg}" + changeLogText += "\n" + commitId.take(7) + " - " + commitMsg + } + } + return changeLogText + } + def changelog = getChangeLogText() + def duration = (System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 + + def slackmsg = "${env.JOB_NAME}-${env.BUILD_NUMBER}: ${currentBuild.result}, Duration: ${duration}, Changelogs: ${changelog}" + + if (currentBuild.result != "SUCCESS") { + slackSend channel: '#pd', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" + } + } +} From 46e04bc1a6560811f1993684c2b5dad6174cee54 Mon Sep 17 00:00:00 2001 From: iamxy Date: Fri, 12 May 2017 15:56:40 +0800 Subject: [PATCH 2/2] Jenkinsfile: take ci pipeline scripts extern --- Jenkinsfile | 245 +--------------------------------------------------- 1 file changed, 2 insertions(+), 243 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8afa5e9a5fcf..f719a3368920 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,249 +4,8 @@ node { def TIDB_TEST_BRANCH = "master" def TIDB_BRANCH = "master" def TIKV_BRANCH = "master" - def UCLOUD_OSS_URL = "http://pingcap-dev.hk.ufileos.com" - env.GOROOT = "/usr/local/go" - env.GOPATH = "/go" - env.PATH = "${env.GOROOT}/bin:/home/jenkins/bin:/bin:${env.PATH}" - - catchError { - stage('Prepare') { - // pd - node('centos7_build') { - def ws = pwd() - dir("go/src/github.com/pingcap/pd") { - // checkout - checkout scm - // build - sh "GOPATH=${ws}/go:$GOPATH make" - } - stash includes: "go/src/github.com/pingcap/pd/**", name: "pd" - } - - // tidb - node('centos7_build') { - def ws = pwd() - dir("go/src/github.com/pingcap/tidb") { - // checkout - git changelog: false, credentialsId: 'github-iamxy-ssh', poll: false, url: 'git@github.com:pingcap/tidb.git', branch: "${TIDB_BRANCH}" - sh "GOPATH=${ws}/go:$GOPATH make parser" - def tidb_sha1 = sh(returnStdout: true, script: "curl ${UCLOUD_OSS_URL}/refs/pingcap/tidb/${TIDB_BRANCH}/centos7/sha1").trim() - sh "curl ${UCLOUD_OSS_URL}/builds/pingcap/tidb/${tidb_sha1}/centos7/tidb-server.tar.gz | tar xz" - } - stash includes: "go/src/github.com/pingcap/tidb/**", name: "tidb" - } - - // tidb-test - dir("go/src/github.com/pingcap/tidb-test") { - // checkout - git changelog: false, credentialsId: 'github-iamxy-ssh', poll: false, url: 'git@github.com:pingcap/tidb-test.git', branch: "${TIDB_TEST_BRANCH}" - } - stash includes: "go/src/github.com/pingcap/tidb-test/**", name: "tidb-test" - - // tikv - def tikv_sha1 = sh(returnStdout: true, script: "curl ${UCLOUD_OSS_URL}/refs/pingcap/tikv/${TIKV_BRANCH}/centos7/sha1").trim() - sh "curl ${UCLOUD_OSS_URL}/builds/pingcap/tikv/${tikv_sha1}/centos7/tikv-server.tar.gz | tar xz" - - unstash 'pd' - sh "cp go/src/github.com/pingcap/pd/bin/pd-server bin/pd-server && rm -rf go/src/github.com/pingcap/pd" - - stash includes: "bin/**", name: "binaries" - } - - stage('Test') { - def tests = [:] - - tests["PD Test"] = { - node("test") { - def ws = pwd() - deleteDir() - unstash 'pd' - - dir("go/src/github.com/pingcap/pd") { - sh "GOPATH=${ws}/go:$GOPATH make test" - } - } - } - - def run_integration_ddl_test = { ddltest -> - def ws = pwd() - deleteDir() - unstash 'tidb' - unstash 'tidb-test' - unstash 'binaries' - - try { - sh """ - killall -9 ddltest_tidb-server || true - killall -9 tikv-server || true - killall -9 pd-server || true - bin/pd-server --name=pd --data-dir=pd &>pd_ddl_test.log & - sleep 10 - bin/tikv-server --pd-endpoints=127.0.0.1:2379 --data-dir=tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 &>tikv_ddl_test.log & - sleep 10 - """ - - timeout(10) { - dir("go/src/github.com/pingcap/tidb-test") { - sh """ - ln -s tidb/_vendor/src ../vendor - cp ${ws}/go/src/github.com/pingcap/tidb/bin/tidb-server ddl_test/ddltest_tidb-server - cd ddl_test && GOPATH=${ws}/go:$GOPATH ./run-tests.sh -check.f='${ddltest}' - """ - } - } - } catch (err) { - throw err - } finally { - sh "killall -9 ddltest_tidb-server || true" - sh "killall -9 tikv-server || true" - sh "killall -9 pd-server || true" - } - } - - tests["Integration DDL Insert Test"] = { - node("test") { - run_integration_ddl_test('TestDDLSuite.TestSimple.*Insert') - } - } - - tests["Integration DDL Update Test"] = { - node("test") { - run_integration_ddl_test('TestDDLSuite.TestSimple.*Update') - } - } - - tests["Integration DDL Delete Test"] = { - node("test") { - run_integration_ddl_test('TestDDLSuite.TestSimple.*Delete') - } - } - - tests["Integration DDL Other Test"] = { - node("test") { - run_integration_ddl_test('TestDDLSuite.TestSimp(le\$|leMixed|leInc)') - } - } - - tests["Integration DDL Column and Index Test"] = { - node("test") { - run_integration_ddl_test('TestDDLSuite.Test(Column|Index)') - } - } - - tests["Integration Connection Test"] = { - node("test") { - def ws = pwd() - deleteDir() - unstash 'tidb' - unstash 'tidb-test' - unstash 'binaries' - - try { - sh """ - killall -9 tikv-server || true - killall -9 pd-server || true - bin/pd-server --name=pd --data-dir=pd &>pd_conntest.log & - sleep 10 - bin/tikv-server --pd-endpoints=127.0.0.1:2379 --data-dir=tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 &>tikv_conntest.log & - sleep 10 - """ - - dir("go/src/github.com/pingcap/tidb") { - sh """ - GOPATH=`pwd`/_vendor:${ws}/go:$GOPATH CGO_ENABLED=1 go test --args with-tikv store/tikv/*.go - """ - } - } catch (err) { - throw err - } finally { - sh "killall -9 tikv-server || true" - sh "killall -9 pd-server || true" - } - } - } - - def run_integration_other_test = { mytest -> - def ws = pwd() - deleteDir() - unstash 'tidb' - unstash 'tidb-test' - unstash 'binaries' - - try { - sh """ - killall -9 tikv-server || true - killall -9 pd-server || true - bin/pd-server --name=pd --data-dir=pd &>pd_${mytest}.log & - sleep 10 - bin/tikv-server --pd-endpoints=127.0.0.1:2379 --data-dir=tikv --addr=0.0.0.0:20160 --advertise-addr=127.0.0.1:20160 &>tikv_${mytest}.log & - sleep 10 - """ - - dir("go/src/github.com/pingcap/tidb-test") { - sh """ - ln -s tidb/_vendor/src ../vendor - GOPATH=${ws}/go:$GOPATH TIKV_PATH='127.0.0.1:2379' TIDB_TEST_STORE_NAME=tikv make ${mytest} - """ - } - } catch (err) { - throw err - } finally { - sh "killall -9 tikv-server || true" - sh "killall -9 pd-server || true" - } - } - - tests["Integration TiDB Test"] = { - node('test') { - run_integration_other_test('tidbtest') - } - } - - tests["Integration MySQL Test"] = { - node("test") { - run_integration_other_test('mysqltest') - } - } - - tests["Integration GORM Test"] = { - node("test") { - run_integration_other_test('gormtest') - } - } - - tests["Integration Go SQL Test"] = { - node("test") { - run_integration_other_test('gosqltest') - } - } - - parallel tests - } - - currentBuild.result = "SUCCESS" - } - - stage('Summary') { - def getChangeLogText = { - def changeLogText = "" - for (int i = 0; i < currentBuild.changeSets.size(); i++) { - for (int j = 0; j < currentBuild.changeSets[i].items.length; j++) { - def commitId = "${currentBuild.changeSets[i].items[j].commitId}" - def commitMsg = "${currentBuild.changeSets[i].items[j].msg}" - changeLogText += "\n" + commitId.take(7) + " - " + commitMsg - } - } - return changeLogText - } - def changelog = getChangeLogText() - def duration = (System.currentTimeMillis() - currentBuild.startTimeInMillis) / 1000 - - def slackmsg = "${env.JOB_NAME}-${env.BUILD_NUMBER}: ${currentBuild.result}, Duration: ${duration}, Changelogs: ${changelog}" - - if (currentBuild.result != "SUCCESS") { - slackSend channel: '#pd', color: 'danger', teamDomain: 'pingcap', tokenCredentialId: 'slack-pingcap-token', message: "${slackmsg}" - } + fileLoader.withGit('git@github.com:pingcap/SRE.git', 'master', 'github-iamxy-ssh', '') { + fileLoader.load('jenkins/ci/pingcap_pd_branch.groovy').call(TIDB_TEST_BRANCH, TIDB_BRANCH, TIKV_BRANCH) } }