Skip to content
Permalink
Branch: master
Find file Copy path
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
511 lines (447 sloc) 24.4 KB
#!/usr/bin/env groovy
import hudson.model.*
import hudson.EnvVars
import groovy.json.JsonSlurper
import groovy.json.JsonBuilder
import groovy.json.JsonOutput
import jenkins.plugins.http_request.*
import java.net.URL
/**
This is an example Jenkins Pipeline Script that runs a CI process against COBOL Code. This pipeline is designed to be triggered from ISPW
on the promotion of code from a Test level in a controlled level. The pipeline runs a series of quality checks on the
promoted code to ensure that it meets the quality standards that an organization defined in SonarQube.
This Pipeline uses the following Jenkins Plugins
Compuware Common Configuration Plugin - https://plugins.jenkins.io/compuware-common-configuration
Compuware Source Code Download for Endevor, PDS, and ISPW Plugin - https://plugins.jenkins.io/compuware-scm-downloader
Compuware Topaz for Total Test Plugin - https://plugins.jenkins.io/compuware-topaz-for-total-test
Compuware Xpediter Code Coverage Plugin - https://plugins.jenkins.io/compuware-xpediter-code-coverage
Pipeline Utilities Plugin - https://plugins.jenkins.io/pipeline-utility-steps
SonarQube Scanner Plugin - https://plugins.jenkins.io/sonar
XebiaLabs XL Release Plugin - https://plugins.jenkins.io/xlrelease-plugin
This Pipeline Requires the below Parameters to be defined in the Jenkins Job
The Jenkins Parameters can be supplied by a ISPW webhook by defining a webhook like the example below.
Please note that the assignment is not currently available in the webhook, but will be added in a future release.
http://<<your jenkins server>>/job/<<you jenkins job>>/buildWithParameters?ISPW_Stream=$$stream$$&ISPW_Container=$$release$$&ISPW_Src_Level=$$level$$&SetId=$$setID$$&ISPW_Release=$$release$$&Owner=$$owner$$
ISPW Webhook Parameter List below
@param ISPW_Stream - ISPW Stream that had the code promotion
@param ISPW_Container - ISPW Container that had the code promotion
@param ISPW_Container_Type - Type of ISPW Container that had the code promotion, 0 - Assignment, 1 - Release, 3 - Set
@param ISPW_Release - The ISPW Release Value that will be passed to XL Release
@param ISPW_Src_Level - ISPW Level that code was promoted from
@param ISPW_Owner - The ISPW Owner value from the ISPW Set that was created for the promotion
The Pipeline also takes the following parameters from the Jenkins Job
@param CES_Token - CES Personal Access Token. These are configured in Compuware Enterprise Services / Security / Personal Access Tokens
@param HCI_Conn_ID - HCI Connection ID configured in the Compuware Common Configuration Plugin. Use Pipeline Syntax Generator to determine this value.
@param HCI_Token - The ID of the Jenkins Credential for the TSO ID that will used to execute the pipeline
@param CES_Connection - The URL of Compuware Enterprise Services
@param CC_repository - The Compuware Xpediter Code Coverage Repository that the Pipeline will use
@param Git_Project - Github project/user used to store the Topaz for Total Test Projects
*/
/**
Below is a list of parameters that is hardcoded into the Pipeline
@param Git_Credentials - Jenkins credentials for logging into git
@param Git_URL - Url that will be used in various git commands
@param Git_TTT_Repo - Git repo that contains Topaz for Total Test Projects
@param Git_Branch - Git branch to be used by the pipeline
@param SQ_Scanner_Name - Name of SonarQube Scanner installation in "Manage Jenkins" -> "Global Tool Configuration" -> "SonarQube Scanner Installations"
@param SQ_Server_Name - Name of SonarQube Server in "Manage Jenkins" -> "Configure System" -> "Sonar Qube servers"
@param MF_Source - directory that contains cobol source downloaded from ISPW
@param XLR_Template - XL Release template to trigger at the end of the Jenkins workflow
@param XLR_User - XL Release user ID. Configured in Jenkins/Manage Jenkins/Configure System/XL Release credentials
@param TTT_Folder - Folder to download TTT projects from GitHub into, i.e. store all TTT projects into one folder
@param ISPW_URL - URL to the ISPW Rest API
@param ISPW_Runtime - ISPW Runtime configuration
*/
String Git_Credentials = "github"
String Git_URL = "https://github.com/${Git_Project}"
String Git_TTT_Repo = "${ISPW_Stream}_${ISPW_Application}_Unit_Tests.git"
String Git_Branch = "master"
String SQ_Scanner_Name = "scanner"
String SQ_Server_Name = "localhost"
String SQ_Project = "${JOB_NAME}"
String MF_Source = "MF_Source"
String XLR_Template = "A Release from Jenkins"
String XLR_User = "admin"
String TTT_Folder = "tests"
String ISPW_URL = "http://cwcc.compuware.com:2020"
String ISPW_Runtime = "ispw"
/*
Map containing ISPW Owners (TSO Users) to eMail addresses
For sending mail when the Quality Gate fails
*/
Map mailRecipientMap = ["HDDRXM0":"ralph.nuesse@compuware.com"]
/**
Helper Methods for the Pipeline Script
*/
/**
Determine the ISPW Path Number for use in Total Test
@param Level - Level Parameter is the Level returned in the ISPW Webhook
*/
def String getPathNum(String Level)
{
return Level.charAt(Level.length() - 1)
}
/**
Wrapper around the Git Plugin's Checkout Method
@param URL - URL for the git server
@param Branch - The branch that will be checked out of git
@param Credentials - Jenkins credentials for logging into git
@param Folder - Folder relative to the workspace that git will check out files into
*/
def gitcheckout(String URL, String Branch, String Credentials, String Folder)
{
println "Scenario " + URL
println "Scenario " + Branch
println "Scenario " + Credentials
checkout changelog: false, poll: false,
scm: [$class: 'GitSCM',
branches: [[name: "*/${Branch}"]],
doGenerateSubmoduleConfigurations: false,
extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: "${Folder}"]],
submoduleCfg: [],
userRemoteConfigs: [[credentialsId: "${Credentials}", name: 'origin', url: "${URL}"]]]
}
/*
Receive a response from an "Get Tasks in Set"-httpRequest and build a list of task IDs
*/
/*
"@NonCPS" is required to tell Jenkins that the objects in the method do not need to survive a Jenkins re-start
This is necessary because JsonSlurper uses non serializable classes, which will leed to exceptions when not used
in methods in a @NonCPS section
*/
@NonCPS
def ArrayList getSetTaskIdList(ResponseContentSupplier response, String level)
{
def jsonSlurper = new JsonSlurper()
def returnList = []
def resp = jsonSlurper.parseText(response.getContent())
if(resp.message != null)
{
echo "Resp: " + resp.message
error
}
else
{
def taskList = resp.tasks
taskList.each
{
if(it.moduleType == 'COB' && it.level == level)
{
returnList.add(it.taskId)
}
}
}
return returnList
}
/*
Receive a list of task IDs and the response of an "List tasks of a Release"-httpRequest to build a list of Assignments
that are contained in both the Task Id List and the List of Tasks in the Release
*/
/*
"@NonCPS" is required to tell Jenkins that the objects in the method do not need to survive a Jenkins re-start
This is necessary because JsonSlurper uses non serializable classes, which will leed to exceptions when not used
in methods in a @NonCPS section
*/
@NonCPS
def ArrayList getAssigmentList(ArrayList taskIds, ResponseContentSupplier response)
{
def jsonSlurper = new JsonSlurper()
def returnList = []
def resp = jsonSlurper.parseText(response.getContent())
if(resp.message != null)
{
echo "Resp: " + resp.message
error
}
else
{
def taskList = resp.tasks
taskList.each
{
if(taskIds.contains(it.taskId))
{
if(!(returnList.contains(it.container)))
{
returnList.add(it.container)
}
}
}
}
return returnList
}
// One node with several stages => All variables are local to the node and available to all stages
node{
// Determine the current ISPW Path and Level that the code Promotion is from
def PathNum = getPathNum(ISPW_Src_Level)
// Use the Path Number to determine the right Runner JCL to use (different STEPLIB concatenations)
def TTT_Jcl = "Runner_PATH" + PathNum + ".jcl"
// Also set the Level that the code currently resides in
def ISPW_Target_Level = "QA" + PathNum
def mailRecipient = mailRecipientMap[(ISPW_Owner.toUpperCase())]
/*
This stage can be used is you want to clean out the workspace from any previously downloaded source from ISPW.
This stage shouldn't be necessary in the ordinary execution of the pipeline
*/
/*
stage("clean previously downloaded source")
{
// Clean out any previously downloaded source
dir(".\\")
{
deleteDir()
}
}
*/
/*************************************************************************************************************/
// Build a list of Assignments based on a Set
// Use httpRequest to get all Tasks for the Set
def ResponseContentSupplier response1
def ResponseContentSupplier response2
def ResponseContentSupplier response3
withCredentials(
[string(credentialsId: "${CES_Token}", variable: 'cesToken')]
)
{
response1 = steps.httpRequest(
url: "${ISPW_URL}/ispw/${ISPW_Runtime}/sets/${ISPW_Container}/tasks",
httpMode: 'GET',
consoleLogResponseBody: false,
customHeaders: [[
maskValue: true,
name: 'authorization',
value: "${cesToken}"
]]
)
}
// Use method getSetTaskIdList to extract the list of Task IDs from the response of the httpRequest
def setTaskIdList = getSetTaskIdList(response1, ISPW_Target_Level)
// Use httpRequest to get all Assignments for the Release
// Need to use two separate objects to store the responses for the httpRequests,
// otherwise the script will fail with a NotSerializable Exception
withCredentials(
[string(credentialsId: "${CES_Token}", variable: 'cesToken')]
)
{
response2 = steps.httpRequest(
url: "${ISPW_URL}/ispw/${ISPW_Runtime}/releases/${ISPW_Release}/tasks",
consoleLogResponseBody: false,
customHeaders: [[
maskValue: true,
name: 'authorization',
value: "${cesToken}"
]]
)
}
// Use method getAssigmentList to get all Assignments from the Release,
// that belong to Tasks in the Set
// If the Sonar Quality Gate fails, these Assignments will be regressed
def assignmentList = getAssigmentList(setTaskIdList, response2)
/*************************************************************************************************************/
def CC_DDIO_Override = "SALESSUP.${ISPW_Application}.${ISPW_Target_Level}.SSD"
/*
This stage is used to retrieve source from ISPW
*/
stage("Retrieve Code From ISPW")
{
//Retrieve the code from ISPW that has been promoted
checkout([
$class: 'IspwContainerConfiguration',
componentType: '', // optional filter for component types in ISPW
connectionId: "${HCI_Conn_ID}",
credentialsId: "${HCI_Token}",
containerName: "${ISPW_Container}",
containerType: "${ISPW_Container_Type}", // 0-Assignment 1-Release 2-Set
ispwDownloadAll: false, // false will not download files that exist in the workspace and haven't previous changed
serverConfig: '', // ISPW runtime config. if blank ISPW will use the default runtime config
serverLevel: ''
]) // level to download the components from
}
/*
This stage is used to retrieve Topaz for Total Tests from Git
*/
stage("Retrieve Tests")
{
//Retrieve the Tests from Github that match that ISPWW Stream and Application
Git_URL = "${Git_URL}/${Git_TTT_Repo}"
//call gitcheckout wrapper function
gitcheckout(Git_URL, Git_Branch, Git_Credentials, TTT_Folder)
}
// findFiles method requires the "Pipeline Utilities Plugin"
// Get all testscenario files in the current workspace into an array
def TTTListOfScenarios = findFiles(glob: '**/*.testscenario')
// Get all Cobol Sources in the MF_Source folder into an array
def ListOfSources = findFiles(glob: "**/${ISPW_Application}/${MF_Source}/*.cbl")
// Define a empty array for the list of programs
def ListOfPrograms = []
// Determine program names for each source member
ListOfSources.each
{
// The split method uses regex to search for patterns, therefore
// Backslashes, Dots and Underscores which mean certain patterns in regex need to be escaped
// The backslash in Windows paths is duplicated in Java, therefore it need to be escaped twice
// Trim ./cbl from the Source members to populate the array of program names
ListOfPrograms.add(it.name.trim().split("\\.")[0])
}
/*
This stage executes any Total Test Projects related to the mainframe source that was downloaded
*/
stage("Execute related Unit Tests")
{
// Loop through all downloaded Topaz for Total Test scenarios
TTTListOfScenarios.each
{
// Get root node of the path, i.e. the name of the Total Test project
def TTTScenarioPath = it.path // Fully qualified name of the Total Test Scenario file
def TTTProjectName = it.path.trim().split("\\\\")[0] + "\\"+ it.path.trim().split("\\\\")[1] // Total Test Project name is the root folder of the full path to the testscenario
def TTTScenarioFullName = it.name // Get the full name of the testscenario file i.e. "name.testscenario"
def TTTScenarioName = it.name.trim().split("\\.")[0] // Get the name of the scenario file without ".testscenario"
def TTTScenarioTarget = TTTScenarioName.split("\\_")[0] // Target Program will be the first part of the scenario name (convention)
// For each of the scenarios walk through the list of source files and determine if the target matches one of the programs
// In that case, execute the unit test. Determine if the program name matches the target of the Total Test scenario
if(ListOfPrograms.contains(TTTScenarioTarget))
{
// Log which
println "*************************"
println "Scenario " + TTTScenarioFullName
println "Path " + TTTScenarioPath
println "Project " + TTTProjectName
println "*************************"
step([
$class: 'TotalTestBuilder',
ccClearStats: false, // Clear out any existing Code Coverage stats for the given ccSystem and ccTestId
ccRepo: "${CC_repository}",
ccSystem: "${ISPW_Application}",
ccTestId: "${BUILD_NUMBER}", // Jenkins environment variable, resolves to build number, i.e. #177
credentialsId: "${HCI_Token}",
deleteTemp: true, // (true|false) Automatically delete any temp files created during the execution
hlq: '', // Optional - high level qualifier used when allocation datasets
connectionId: "${HCI_Conn_ID}",
jcl: "${TTT_Jcl}", // Name of the JCL file in the Total Test Project to execute
projectFolder: "${TTTProjectName}", // Name of the Folder in the file system that contains the Total Test Project.
testSuite: "${TTTScenarioFullName}",// Name of the Total Test Scenario to execute
useStubs: true
]) // (true|false) - Execute with or without stubs
}
}
// Process the Total Test Junit result files into Jenkins
junit allowEmptyResults: true,
keepLongStdio: true,
testResults: "TTTUnit/*.xml"
}
/*
This stage retrieve Code Coverage metrics from Xpediter Code Coverage for the test executed in the Pipeline
*/
stage("Collect Coverage Metrics")
{
// Code Coverage needs to match the code coverage metrics back to the source code in order for them to be loaded in SonarQube
// The source variable is the location of the source that was downloaded from ISPW
def String sources="${ISPW_Application}\\${MF_Source}"
// The Code Coverage Plugin passes it's primary configuration in the string or a file
def ccproperties = 'cc.sources=' + sources + '\rcc.repos=' + CC_repository + '\rcc.system=' + ISPW_Application + '\rcc.test=' + BUILD_NUMBER
step([
$class: 'CodeCoverageBuilder',
analysisProperties: ccproperties, // Pass in the analysisProperties as a string
analysisPropertiesPath: '', // Pass in the analysisProperties as a file. Not used in this example
connectionId: "${HCI_Conn_ID}",
credentialsId: "${HCI_Token}"
])
}
/*
This stage pushes the Source Code, Test Metrics and Coverage metrics into SonarQube and then checks the status of the SonarQube Quality Gate.
If the SonarQube quality date fails, the Pipeline fails and stops
*/
stage("Check SonarQube Quality Gate")
{
// Requires SonarQube Scanner 2.8+
// Retrieve the location of the SonarQube Scanner.
def scannerHome = tool "${SQ_Scanner_Name}";
withSonarQubeEnv("${SQ_Server_Name}") // 'localhost' is the name of the SonarQube server defined in Jenkins / Configure Systems / SonarQube server section
{
// Finds all of the Total Test results files that will be submitted to SonarQube
def TTTListOfResults = findFiles(glob: 'TTTSonar/*.xml') // Total Test SonarQube result files are stored in TTTSonar directory
// Build the sonar testExecutionReportsPaths property
// Start will the property itself
def SQ_TestResult = "-Dsonar.testExecutionReportPaths="
// Loop through each result Total Test results file found
TTTListOfResults.each
{
def TTTResultName = it.name // Get the name of the Total Test results file
SQ_TestResult = SQ_TestResult + "TTTSonar/" + it.name + ',' // Append the results file to the property
}
// Build the rest of the SonarQube Scanner Properties
// Test and Coverage results
def SQ_Scanner_Properties = " -Dsonar.tests=tests ${SQ_TestResult} -Dsonar.coverageReportPaths=Coverage/CodeCoverage.xml"
// SonarQube project to load results into
SQ_Scanner_Properties = SQ_Scanner_Properties + " -Dsonar.projectKey=${JOB_NAME} -Dsonar.projectName=${JOB_NAME} -Dsonar.projectVersion=1.0"
// Location of the Cobol Source Code to scan
SQ_Scanner_Properties = SQ_Scanner_Properties + " -Dsonar.sources=${ISPW_Application}\\MF_Source"
// Location of the Cobol copybooks to scan
SQ_Scanner_Properties = SQ_Scanner_Properties + " -Dsonar.cobol.copy.directories=${ISPW_Application}\\MF_Source"
// File extensions for Cobol and Copybook files. The Total Test files need that contain tests need to be defined as cobol for SonarQube to process the results
SQ_Scanner_Properties = SQ_Scanner_Properties + " -Dsonar.cobol.file.suffixes=cbl,testsuite,testscenario,stub -Dsonar.cobol.copy.suffixes=cpy -Dsonar.sourceEncoding=UTF-8"
// Call the SonarQube Scanner with properties defined above
bat "${scannerHome}/bin/sonar-scanner" + SQ_Scanner_Properties
}
// Wait for the results of the SonarQube Quality Gate
timeout(time: 2, unit: 'MINUTES') {
// Wait for webhook call back from SonarQube. SonarQube webhook for callback to Jenkins must be configured on the SonarQube server.
def qg = waitForQualityGate()
// Evaluate the status of the Quality Gate
if (qg.status != 'OK')
{
echo "Sonar quality gate failure: ${qg.status}"
echo "Pipeline will be aborted and ISPW Assignment(s) will be regressed"
for(int i = 0; i < assignmentList.size(); i++)
{
echo "Regress Assignment ${assignmentList[0].toString()}, Level ${ISPW_Target_Level}"
def requestBodyParm = '''{
"runtimeConfiguration": "''' + ISPW_Runtime + '''"
}'''
withCredentials(
[string(credentialsId: "${CES_Token}", variable: 'cesToken')]
)
{
response3 = steps.httpRequest(
url: "${ISPW_URL}/ispw/${ISPW_Runtime}/assignments/${assignmentList[i].toString()}/tasks/regress?level=${ISPW_Target_Level}",
httpMode: 'POST',
consoleLogResponseBody: true,
contentType: 'APPLICATION_JSON',
requestBody: requestBodyParm,
customHeaders: [[
maskValue: true,
name: 'authorization',
value: "${cesToken}"
]]
)
}
}
// Email
emailext subject: '$DEFAULT_SUBJECT',
body: '$DEFAULT_CONTENT',
replyTo: '$DEFAULT_REPLYTO',
to: "${mailRecipient}"
error "Exiting Pipeline" // Exit the pipeline with an error if the SonarQube Quality Gate is failing
}
}
}
/*
This stage triggers a XL Release Pipeline that will move code into the high levels in the ISPW Lifecycle
*/
stage("Start release in XL Release")
{
// Use the Path Number to determine what QA Path to Promote the code from in ISPW. This example has seperate QA paths in ISPW Lifecycle (i.e. DEV1->QA1->STG->PRD / DEV2->QA2->STG->PRD)
def XLRPath = "QA" + PathNum
// Trigger XL Release Jenkins Plugin to kickoff a Release
xlrCreateRelease(
releaseTitle: 'A Release for $BUILD_TAG',
serverCredentials: "${XLR_User}",
startRelease: true,
template: "${XLR_Template}",
variables: [
[propertyName:'ISPW_Dev_level', propertyValue: "${ISPW_Target_Level}"], // Level in ISPW that the Code resides currently
[propertyName: 'ISPW_RELEASE_ID', propertyValue: "${ISPW_Release}"], // ISPW Release value from the ISPW Webhook
[propertyName: 'CES_Token', propertyValue: "${CES_Token}"]
]
)
}
}
You can’t perform that action at this time.