diff --git a/Jenkinsfile b/Jenkinsfile index 3e2f13e34e..2007c7666a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,17 +1,24 @@ @Library('shared-libraries') _ import groovy.json.JsonSlurper -def gitDataHubRepo="https://github.com/SameeraPriyathamTadikonda/marklogic-data-hub.git" -def JAVA_HOME="~/java/jdk1.8.0_72" -def GRADLE_USER_HOME="/.gradle" -def MAVEN_HOME="/usr/local/maven" +import groovy.json.JsonSlurperClassic def JIRA_ID=""; +def commitMessage=""; def prResponse=""; def prNumber; +def githubAPIUrl="https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub" pipeline{ agent none; options { checkoutToSubdirectory 'data-hub' } + environment{ + JAVA_HOME_DIR="~/java/jdk1.8.0_72" + GRADLE_DIR="/.gradle" + MAVEN_HOME="/usr/local/maven" + } + parameters{ + string(name: 'Email', defaultValue: 'stadikon@marklogic.com', description: 'Who should I say send the email to?') + } stages{ stage('Build-datahub'){ agent { label 'dhfLinuxAgent'} @@ -24,15 +31,16 @@ pipeline{ } } println(BRANCH_NAME) - sh 'echo '+JAVA_HOME+'export '+JAVA_HOME+' export $WORKSPACE/data-hub'+GRADLE_USER_HOME+'export '+MAVEN_HOME+'export PATH=$PATH:$MAVEN_HOME/bin; cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew build -x test -Pskipui=true;' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean --stacktrace;./gradlew build -x test -Pskipui=true;' archiveArtifacts artifacts: 'data-hub/marklogic-data-hub/build/libs/* , data-hub/ml-data-hub-plugin/build/libs/* , data-hub/quick-start/build/libs/', onlyIfSuccessful: true } } stage('Unit-Tests'){ agent { label 'dhfLinuxAgent'} steps{ copyRPM 'Latest' - setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' - sh 'echo '+JAVA_HOME+'export '+JAVA_HOME+' export $WORKSPACE/data-hub'+GRADLE_USER_HOME+'export '+MAVEN_HOME+'export PATH=$PATH:$MAVEN_HOME/bin; cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.flow.* -Pskipui=true' + //setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + setUpML '/space/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.processes.ProcessManagerTest -Pskipui=true' junit '**/TEST-*.xml' script{ if(env.CHANGE_TITLE){ @@ -44,11 +52,29 @@ pipeline{ post{ success { println("Unit Tests Completed") - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,'Unit Tests for $BRANCH_NAME Passed' + script{ + def email; + if(env.CHANGE_AUTHOR){ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + email=getEmailFromGITUser author + }else{ + email=Email + } + sendMail email,'Check: ${BUILD_URL}/console',false,'Unit Tests for $BRANCH_NAME Passed' + } } failure { println("Unit Tests Failed") - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,'Unit Tests for $BRANCH_NAME Failed' + script{ + def email; + if(env.CHANGE_AUTHOR){ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + email=getEmailFromGITUser author + }else{ + email=Email + } + sendMail email,'Check: ${BUILD_URL}/console',false,'Unit Tests for $BRANCH_NAME Failed' + } } } } @@ -66,8 +92,11 @@ pipeline{ println("Automated PR") sh 'exit 0' }else{ - - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,'Waiting for code review $BRANCH_NAME ' + script{ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + def email=getEmailFromGITUser author + sendMail email,'Check: ${BUILD_URL}/console',false,'Waiting for code review $BRANCH_NAME ' + } try{ timeout(time:5, unit:'MINUTES') { input message:'Review Done?' @@ -89,13 +118,13 @@ pipeline{ retry(5){ withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { script{ - - def response = sh (returnStdout: true, script:'''curl -u $Credentials --header "application/vnd.github.merge-info-preview+json" "https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub/pulls/$CHANGE_ID" | grep '"mergeable_state":' | cut -d ':' -f2 | cut -d ',' -f1 | tr -d '"' ''') + JIRA_ID=env.CHANGE_TITLE.split(':')[0] + def response = sh (returnStdout: true, script:'''curl -u $Credentials --header "application/vnd.github.merge-info-preview+json" "'''+githubAPIUrl+'''/pulls/$CHANGE_ID" | grep '"mergeable_state":' | cut -d ':' -f2 | cut -d ',' -f1 | tr -d '"' ''') response=response.trim(); println(response) if(response.equals("clean")){ println("merging can be done") - sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"Merge pull request\"}' -u $Credentials https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub/pulls/$CHANGE_ID/merge | tail -1 > mergeResult.txt" + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: merging PR\"}' -u $Credentials "+ githubAPIUrl+"/pulls/$CHANGE_ID/merge | tail -1 > mergeResult.txt" def mergeResult = readFile('mergeResult.txt').trim() if(mergeResult==200){ println("Merge successful") @@ -108,7 +137,7 @@ pipeline{ throw new Exception("Waiting for all the status checks to pass"); }else if(response.equals("unstable")){ println("retry unstable") - sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"Merge pull request\"}' -u $Credentials https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub/pulls/$CHANGE_ID/merge | tail -1 > mergeResult.txt" + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: merging PR\"}' -u $Credentials "+githubAPIUrl+"/pulls/$CHANGE_ID/merge | tail -1 > mergeResult.txt" def mergeResult = readFile('mergeResult.txt').trim() println("Result is"+ mergeResult) }else{ @@ -124,57 +153,67 @@ pipeline{ success { println("Merge Successful") script{ - if(env.CHANGE_TITLE){ - def transitionInput =[transition: [id: '31']] - JIRA_ID=env.CHANGE_TITLE.split(':')[0] - jiraTransitionIssue idOrKey: JIRA_ID, input: transitionInput, site: 'JIRA' - } - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,' $BRANCH_NAME is Merged' + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + def email=getEmailFromGITUser author + sendMail email,'Check: ${BUILD_URL}/console',false,' $BRANCH_NAME is Merged' } } failure { println("Retried 5times") - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,' $BRANCH_NAME Cannot be Merged' + script{ + def author=env.CHANGE_AUTHOR.toString().trim().toLowerCase() + def email=getEmailFromGITUser author + sendMail email,'Check: ${BUILD_URL}/console',false,' $BRANCH_NAME Cannot be Merged' + } } } } - stage('Integration Tests'){ + stage('E2E Single Tests'){ + when { + branch 'FeatureBranch' + } agent { label 'dhfLinuxAgent'} steps{ copyRPM 'Latest' - setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' - sh 'echo '+JAVA_HOME+'export '+JAVA_HOME+' export $WORKSPACE/data-hub'+GRADLE_USER_HOME+'export '+MAVEN_HOME+'export PATH=$PATH:$MAVEN_HOME/bin; cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.flow.* -Pskipui=true' + //setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + setUpML '/space/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.processes.ProcessManagerTest -Pskipui=true' junit '**/TEST-*.xml' script{ - if(env.CHANGE_TITLE){ - JIRA_ID=env.CHANGE_TITLE.split(':')[0] + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; jiraAddComment comment: 'Jenkins End-End Unit Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' } - } } post{ success { println("End-End Tests Completed") - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,'End-End Tests for $BRANCH_NAME Passed' + sendMail Email,'Check: ${BUILD_URL}/console',false,'End-End Tests for $BRANCH_NAME Passed' + } failure { println("End-End Tests Failed") - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,'End-End Tests for $BRANCH_NAME Failed' + sendMail Email,'Check: ${BUILD_URL}/console',false,'End-End Tests for $BRANCH_NAME Failed' } } } - stage('Create PR For Integration Branch'){ + stage('Merge PR to Integration Branch'){ when { - changeRequest target: 'FeatureBranch' + branch 'FeatureBranch' beforeAgent true } agent {label 'master'} steps{ withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { script{ - JIRA_ID=env.CHANGE_TITLE.split(':')[0] + //JIRA_ID=env.CHANGE_TITLE.split(':')[0] prResponse = sh (returnStdout: true, script:''' - curl -u $Credentials -X POST -H 'Content-Type:application/json' -d '{\"title\": \"'''+JIRA_ID+''': Automated PR for Integration Branch\" , \"head\": \"FeatureBranch\" , \"base\": \"IntegrationBranch\" }' https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub/pulls ''') + curl -u $Credentials -X POST -H 'Content-Type:application/json' -d '{\"title\": \"Automated PR for Integration Branch\" , \"head\": \"FeatureBranch\" , \"base\": \"IntegrationBranch\" }' '''+githubAPIUrl+'''/pulls ''') println(prResponse) def slurper = new JsonSlurper().parseText(prResponse) println(slurper.number) @@ -182,13 +221,24 @@ pipeline{ } } withCredentials([usernameColonPassword(credentialsId: 'rahul-git', variable: 'Credentials')]) { - sh "curl -u $Credentials -X POST -d '{\"event\": \"APPROVE\"}' https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub/pulls/${prNumber}/reviews" + sh "curl -u $Credentials -X POST -d '{\"event\": \"APPROVE\"}' "+githubAPIUrl+"/pulls/${prNumber}/reviews" } + withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { + script{ + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: Merge pull request\"}' -u $Credentials "+githubAPIUrl+"/pulls/${prNumber}/merge | tail -1 > mergeResult.txt" + def mergeResult = readFile('mergeResult.txt').trim() + if(mergeResult==200){ + println("Merge successful") + }else{ + println("Merge Failed") + } + } + } } post{ success { - println("Automated PR For Integration branch created") + println("Automated PR For Integration branch Completed") } failure { println("Creation of Automated PR Failed") @@ -196,43 +246,92 @@ pipeline{ } } } + stage('Parallel Execution'){ + parallel{ stage('Upgrade Tests'){ + when { + branch 'IntegrationBranch' + beforeAgent true + } agent { label 'dhfLinuxAgent'} - steps{ + steps{ copyRPM 'Latest' - setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' - sh 'echo '+JAVA_HOME+'export '+JAVA_HOME+' export $WORKSPACE/data-hub'+GRADLE_USER_HOME+'export '+MAVEN_HOME+'export PATH=$PATH:$MAVEN_HOME/bin; cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.flow.* -Pskipui=true' + //setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + setUpML '/space/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.processes.ProcessManagerTest -Pskipui=true' junit '**/TEST-*.xml' - script{ - if(env.CHANGE_TITLE){ - JIRA_ID=env.CHANGE_TITLE.split(':')[0] + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; jiraAddComment comment: 'Jenkins Upgrade Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' } + } + post{ + success { + println("Upgrade Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'Upgrade Tests for $BRANCH_NAME Passed' + // sh './gradlew publish' + } + failure { + println("Upgrade Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'Upgrade Tests for $BRANCH_NAME Failed' + } + } + } + stage('End2End all Tests'){ + when { + branch 'IntegrationBranch' + beforeAgent true + } + agent { label 'dhfWindowsAgent'} + steps{ + copyRPM 'Latest' + //setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + setUpML '/space/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.processes.ProcessManagerTest -Pskipui=true' + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins End-End All Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' } } post{ success { println("Upgrade Tests Completed") - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,'Upgrade Tests for $BRANCH_NAME Passed' + sendMail Email,'Check: ${BUILD_URL}/console',false,'End-End all Tests for $BRANCH_NAME Passed' + // sh './gradlew publish' } failure { println("Upgrade Tests Failed") - sendMail 'stadikon@marklogic.com','Check: ${BUILD_URL}/console',false,'Upgrade Tests for $BRANCH_NAME Failed' + sendMail Email,'Check: ${BUILD_URL}/console',false,'End-End All Tests for $BRANCH_NAME Failed' } } } - stage('Create PR For Release Branch'){ + } + } + stage('Merge PR to Release Branch'){ when { - changeRequest target: 'IntegrationBranch' + branch 'IntegrationBranch' beforeAgent true } agent {label 'master'} steps{ withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { script{ - JIRA_ID=env.CHANGE_TITLE.split(':')[0] + //JIRA_ID=env.CHANGE_TITLE.split(':')[0] prResponse = sh (returnStdout: true, script:''' - sh "curl -u $Credentials -X POST -H 'Content-Type:application/json' -d '{\"title\": \"'''+JIRA_ID+''': Automated PR for Release Branch\" , \"head\": \"IntegrationBranch\" , \"base\": \"ReleaseBranch\" }' https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub/pulls"''') + curl -u $Credentials -X POST -H 'Content-Type:application/json' -d '{\"title\": \"Automated PR for Integration Branch\" , \"head\": \"IntegrationBranch\" , \"base\": \"ReleaseBranch\" }' '''+githubAPIUrl+'''/pulls ''') println(prResponse) def slurper = new JsonSlurper().parseText(prResponse) println(slurper.number) @@ -240,8 +339,19 @@ pipeline{ } } withCredentials([usernameColonPassword(credentialsId: 'rahul-git', variable: 'Credentials')]) { - sh "curl -u $Credentials -X POST -d '{\"event\": \"APPROVE\"}' https://api.github.com/repos/SameeraPriyathamTadikonda/marklogic-data-hub/pulls/${prNumber}/reviews" + sh "curl -u $Credentials -X POST -d '{\"event\": \"APPROVE\"}' "+githubAPIUrl+"/pulls/${prNumber}/reviews" } + withCredentials([usernameColonPassword(credentialsId: 'a0ec09aa-f339-44de-87c4-1a4936df44f5', variable: 'Credentials')]) { + script{ + sh "curl -o - -s -w \"\n%{http_code}\n\" -X PUT -d '{\"commit_title\": \"$JIRA_ID: Merge pull request\"}' -u $Credentials "+githubAPIUrl+"/pulls/${prNumber}/merge | tail -1 > mergeResult.txt" + def mergeResult = readFile('mergeResult.txt').trim() + if(mergeResult==200){ + println("Merge successful") + }else{ + println("Merge Failed") + } + } + } } post{ @@ -254,5 +364,46 @@ pipeline{ } } } + stage('Sanity Tests'){ + when { + branch 'ReleaseBranch' + beforeAgent true + } + agent { label 'dhfLinuxAgent'} + steps{ + copyRPM 'Latest' + //setUpML '$WORKSPACE/xdmp/src/Mark*.rpm' + setUpML '/space/Mark*.rpm' + sh 'echo $JAVA_HOME;export JAVA_HOME=`$JAVA_HOME_DIR`;export GRADLE_USER_HOME=$WORKSPACE$GRADLE_DIR;export M2_HOME=$MAVEN_HOME/bin;export PATH=$GRADLE_USR_HOME:$PATH:$MAVEN_HOME/bin;cd $WORKSPACE/data-hub;rm -rf $GRADLE_USER_HOME/caches;./gradlew clean;./gradlew clean;./gradlew :marklogic-data-hub:test --tests com.marklogic.hub.processes.ProcessManagerTest -Pskipui=true' + junit '**/TEST-*.xml' + script{ + commitMessage = sh (returnStdout: true, script:''' + curl -u $Credentials -X GET "'''+githubAPIUrl+'''/git/commits/${GIT_COMMIT}" ''') + def slurper = new JsonSlurperClassic().parseText(commitMessage.toString().trim()) + def commit=slurper.message.toString().trim(); + JIRA_ID=commit.split(("\\n"))[0].split(':')[0].trim(); + JIRA_ID=JIRA_ID.split(" ")[0]; + commitMessage=null; + jiraAddComment comment: 'Jenkins Sanity Test Results For PR Available', idOrKey: JIRA_ID, site: 'JIRA' + } + } + post{ + success { + println("Sanity Tests Completed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'Sanity Tests for $BRANCH_NAME Passed' + script{ + def transitionInput =[transition: [id: '31']] + //JIRA_ID=env.CHANGE_TITLE.split(':')[0] + jiraTransitionIssue idOrKey: JIRA_ID, input: transitionInput, site: 'JIRA' + } + sendMail Email,'Click approve to release',false,'Datahub is ready for Release' + + } + failure { + println("Sanity Tests Failed") + sendMail Email,'Check: ${BUILD_URL}/console',false,'Sanity Tests for $BRANCH_NAME Failed' + } + } + } } } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 3299d0d430..61abaeea1a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,3 +14,4 @@ services: - "8000-8020:8000-8020" volumes: - .:/marklogic-data-hub +