Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 88 additions & 13 deletions Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,24 @@ import com.ibm.jzos.ZFile;
* Version 8 - 2024-07
* - Reworked error management and fixed few glitches
*
* Version 9 - 2024-10
* - Added the following
* a) Fields and code to generate concert manifest linking to
* concertBuildManifestGenerator.groovy
* b) Refactoring to generate SBOM details like SerialNumber from this script
* to be passed to sbomGenerator.groovy and concertBuildManifestGenerator.groovy
************************************************************************************/

// start create & publish package
@Field Properties props = null
def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent
@Field def wdManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/WaziDeployManifestGenerator.groovy"))
@Field def sbomUtilities
@Field def concertManifestGeneratorUtilities
@Field def rc = 0
@Field def sbomSerialNumber
@Field def sbomFileName
@Field def concertBuild

def startTime = new Date()

Expand Down Expand Up @@ -91,15 +101,12 @@ Map<DeployableArtifact, Map> buildOutputsMap = new HashMap<DeployableArtifact, M

// Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed.
def String tarFileLabel = "Default"
// Field to store build number, set to default when its not decipherable from build report
def String buildNumber = "UNKNOWN"

// Object to store scm information for Wazi Deploy Application Manifest file
HashMap<String,String> scmInfo = new HashMap<String, String>()

if (props.generateSBOM && props.generateSBOM.toBoolean()) {
sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy"))
sbomUtilities.initializeSBOM(props.sbomAuthor)
}

// iterate over all build reports to obtain build output
props.buildReportOrder.each { buildReportFile ->
println("** Read build report data from '${buildReportFile}'.")
Expand All @@ -119,6 +126,7 @@ props.buildReportOrder.each { buildReportFile ->
}
if (buildInfo.size() != 0) {
tarFileLabel = buildInfo[0].label
buildNumber = buildInfo[0].label
}

// retrieve the buildResultPropertiesRecord
Expand Down Expand Up @@ -326,7 +334,9 @@ props.buildReportOrder.each { buildReportFile ->
}

// generate scmInfo for Wazi Deploy Application Manifest file
if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) {
if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) ||
(props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) {

if (props.buildReportOrder.size() == 1) {
scmInfo.put("type", "git")
gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl")
Expand All @@ -348,13 +358,30 @@ if (rc == 0) {
println("** There are no build outputs found in all provided build reports. Exiting.")
rc = 0
} else {

// generate SBOM only if build outputs exist
if (props.generateSBOM && props.generateSBOM.toBoolean()) {
sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy"))
sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString()
sbomFileName = "${buildNumber}_sbom.json"
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the file name format rather something that we should declare at the beginning of the script?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks like the buildNumber is retrieved from the build report. So, it might make sense to retain this where it is.

sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber)

}
// Local variables
// Initialize Wazi Deploy Manifest Generator
if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) {
wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest
wdManifestGeneratorUtilities.setScmInfo(scmInfo)
}

// Initialize Concert Build Manifest Generator
if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) {
// Concert Build Manifest

concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy"))
concertManifestGeneratorUtilities.initConcertBuildManifestGenerator()
concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber)
concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit)
}
def String tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar"
def tarFile = "$props.workDir/${tarFileName}"

Expand Down Expand Up @@ -456,7 +483,7 @@ if (rc == 0) {
}

if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) {
sbomUtilities.writeSBOM("$tempLoadDir/sbom.json", props.fileEncoding)
sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding)
}


Expand All @@ -465,6 +492,7 @@ if (rc == 0) {
// wazideploy_manifest.yml is the default name of the manifest file
wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose)
}


if (rc == 0) {

Expand Down Expand Up @@ -567,6 +595,36 @@ if (rc == 0) {

println ("** Upload package to Artifact Repository '$url'.")
artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion)

// generate PackageInfo for Concert Manifest file
if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) {
concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url)
}
}

if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) {
// concert_build_manifest.yaml is the default name of the manifest file

if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) {
concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber)
}
concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose)
println("** Add concert build config yaml to tar file at ${tarFile}")
// Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file
// To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX
def processCmd = [
"sh",
"-c",
"tar rUXf $tarFile concert_build_manifest.yaml"
]

def processRC = runProcess(processCmd, tempLoadDir)
rc = Math.max(rc, processRC)
if (rc == 0) {
println("** Package '${tarFile}' successfully appended with concert manifest yaml.")
} else {
println("*! [ERROR] Error appending '${tarFile}' with concert manifest yaml rc=$rc.")
}
}
}
}
Expand Down Expand Up @@ -643,12 +701,15 @@ def parseInput(String[] cliArgs){
// Wazi Deploy Application Manifest generation
cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.')

// Concert Build Manifest generation
cli.ic(longOpt:'generateConcertBuildManifest', 'Flag indicating to generate and add the IBM Concert Build Manifest file.')

cli.b(longOpt:'branch', args:1, argName:'branch', 'The git branch processed by the pipeline')
cli.a(longOpt:'application', args:1, argName:'application', 'The name of the application')

// Artifact repository options ::
cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifact Repository server. (Optional)')
cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)')
cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)')

// Artifact repository info
cli.au(longOpt:'artifactRepositoryUrl', args:1, argName:'url', 'URL to the Artifact repository server. (Optional)')
Expand Down Expand Up @@ -705,6 +766,7 @@ def parseInput(String[] cliArgs){

// cli overrides defaults set in 'packageBuildOutputs.properties'
props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : props.generateWaziDeployAppManifest
props.generateConcertBuildManifest = (opts.ic) ? 'true' : props.generateConcertBuildManifest
props.addExtension = (opts.ae) ? 'true' : props.addExtension
props.publish = (opts.p) ? 'true' : props.publish
props.generateSBOM = (opts.sbom) ? 'true' : props.generateSBOM
Expand Down Expand Up @@ -802,10 +864,6 @@ def parseInput(String[] cliArgs){
println("*! [ERROR] Missing Artifact Repository Password property. It is required when publishing the package via ArtifactRepositoryHelpers.")
rc = 2
}
if (!props.'artifactRepository.directory') {
println("*! [ERROR] Missing Artifact Repository Directory property. It is required when publishing the package via ArtifactRepositoryHelpers.")
rc = 2
}
}

// assess required options to generate Wazi Deploy application manifest
Expand All @@ -819,6 +877,23 @@ def parseInput(String[] cliArgs){
rc = 2
}
}

// assess required options to generate Concert Build manifest
if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) {
if (!props.branch) {
println("*! [ERROR] Missing branch parameter ('--branch'). It is required for generating the Concert Build Manifest file.")
rc = 2
}
if (!props.publish) {
println("*! [ERROR] Missing publish parameter ('--publish'). It is required for generating the Concert Build Manifest file.")
rc = 2
}
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it a correct assumption that generating the concert build manifest file, will only work when only one BuildReport.json is passed to the PackageBuildOutputsScript? If so, we should validate that in this block.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We will need to discuss this

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done. Added the check. Need some more tests

if (opts.bO || opts.boFile) {
println("*! [ERROR] conflicting parameter ('-bO or -boFile'). IBM Concert Build Manifest file is created with single builds only.")
rc = 2
}
}

return props
}

Expand Down
19 changes: 18 additions & 1 deletion Pipeline/PackageBuildOutputs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ This section provides a more detailed explanation of how the PackageBuildOutputs
6. **(Optional) Publish to Artifact Repository such as JFrog Artifactory or Sonartype Nexus**
1. Publishes the TAR file to the artifact repository based on the given configuration using the ArtifactRepositoryHelpers script. Consider a Nexus RAW, or a Artifactory Generic as the repository type. **Please note**: The ArtifactRepositoryHelpers script is updated for DBB 2.0 and requires to run on JAVA 11. The publishing can be configured to pass in the artifact repository information as well as the path within the repository `directory/[versionName|buildLabel]/tarFileName` via the cli.

7. **(Optional) Generate IBM Concert Build manifest**
1. Based on the collected build outputs information, the IBM Concert Build Manifest file is generated and saved as concert_build_manifest.yaml. This is a feeder file to publish build information into IBM Concert. It will only be generated if both sbom and packaging options are in effect.

Notes:
* The script doesn't manage the deletions of artifacts. Although they are reported in the DBB Build Reports, deletions are not handled by this script.

Expand Down Expand Up @@ -566,9 +569,23 @@ As an example, you can invoke the SBOM generation with the following command:
/usr/lpp/dbb/v2r0/bin/groovyz -cp /u/mdalbin/SBOM/cyclonedx-core-java-8.0.3.jar:/u/mdalbin/SBOM/jackson-annotations-2.16.1.jar:/u/mdalbin/SBOM/jackson-core-2.16.1.jar:/u/mdalbin/SBOM/jackson-databind-2.16.1.jar:/u/mdalbin/SBOM/jackson-dataformat-xml-2.16.1.jar:/u/mdalbin/SBOM/json-schema-validator-1.2.0.jar:/u/mdalbin/SBOM/packageurl-java-1.5.0.jar /u/mdalbin/SBOM/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputsWithSBOM.groovy --workDir /u/ado/workspace/MortgageApplication/feature/consumeRetirementCalculatorServiceImpacts/build-20240312.1/logs --tarFileName MortgageApplication.tar --addExtension -s -sa "David Gilmour <david.gilmour@pinkfloyd.com>"
~~~~

By default, the SBOM file is generated in the `tempPackageDir` and named `sbom.json`.
By default, the SBOM file is generated in the `tempPackageDir` and named `<buildnumber>_sbom.json`.
This way, it is automatically packaged in the TAR file that is created by the script, ensuring the package and its content are not tampered and correctly documented.

## IBM Concert Build manifest generation

This `PackageBuildOutputs.groovy` script is able to generate an IBM Concert Build manifest based on the information contained in the DBB Build Report and the published package information. The output is a YAML file that adheres to IBM Concert Build specification YAML format. The generation of the CycloneDX SBOM is a pre-requisite as the IBM Concert Build manifest will reference the CycloneDX SBOM file for detailed information about the build outputs.

To enable the generation of the IBM Concert Build manifest, the `-ic/--generateConcertBuildManifest` flag must be passed.

As an example, you can invoke the generation of an IBM Concert Build manifest with the following command:

~~~~
/usr/lpp/dbb/v2r0/bin/groovyz -cp /u/mdalbin/SBOM/cyclonedx-core-java-8.0.3.jar:/u/mdalbin/SBOM/jackson-annotations-2.16.1.jar:/u/mdalbin/SBOM/jackson-core-2.16.1.jar:/u/mdalbin/SBOM/jackson-databind-2.16.1.jar:/u/mdalbin/SBOM/jackson-dataformat-xml-2.16.1.jar:/u/mdalbin/SBOM/json-schema-validator-1.2.0.jar:/u/mdalbin/SBOM/packageurl-java-1.5.0.jar /u/mdalbin/SBOM/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputsWithSBOM.groovy --workDir /u/ado/workspace/MortgageApplication/feature/consumeRetirementCalculatorServiceImpacts/build-20240312.1/logs --tarFileName MortgageApplication.tar --addExtension -s -sa "David Gilmour <david.gilmour@pinkfloyd.com>" -ic
~~~~

By default, the IBM Concert Build manifest file is generated in the `tempPackageDir` and named `concert_build_manifest.yaml`.


## Useful reference material

Expand Down
5 changes: 5 additions & 0 deletions Pipeline/PackageBuildOutputs/packageBuildOutputs.properties
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ addExtension=false
# Default: false
generateWaziDeployAppManifest=false

# Boolean setting to define if the IBM Concert Build Manifest file should be generated
# Please note that the cli option `generateConcertBuildManifest` can override this setting and activate it.
# Default: false
generateConcertBuildManifest=false

# Boolean setting to define if SBOM based on cycloneDX should be generated
# Please note that the cli option `generateSBOM` can override this setting and activate it.
# Requires the classpath to contain the cycloneDX libs
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
import groovy.transform.*
import groovy.yaml.YamlBuilder
import com.ibm.dbb.build.report.records.*

/*
* This is a utility method to generate IBM Concert build files
*/

@Field ConcertBuildManifest concertManifest = new ConcertBuildManifest()

def initConcertBuildManifestGenerator() {
// Metadata
concertManifest.concert = new Concertdata()
concertManifest.concert.builds = new ArrayList<Build>()
}

def addBuild(String application, String version, String buildNumber) {
Build build = new Build()
build.repositories = new ArrayList<Repository>()
build.library = new Library()
build.component_name = application
build.number = buildNumber
build.output_file = application + "_sbom.json"
// Metadata information
build.version = version
concertManifest.concert.builds.add(build)
return build
}

def addRepositoryToBuild(Build build, String url, String branch, String shortCommit) {
Repository repository = new Repository()
repository.name = build.component_name
repository.url = url
repository.branch = branch
repository.commit_sha = shortCommit
build.repositories.add(repository)
}

def addLibraryInfoTobuild(Build build, String filename, String url) {
build.library.scope = 'tar'
build.library.name = build.component_name
build.library.filename = filename
build.library.version = build.version
build.library.url = url
}

def addSBOMInfoToBuild(Build build, String sbomFileName, String sbomSerialNumber) {
build.library.cyclonedx_bom_link = new SBOMInfo()
build.library.cyclonedx_bom_link.file = sbomFileName
build.library.cyclonedx_bom_link.data = new SBOMdata()
build.library.cyclonedx_bom_link.data.serial_number = sbomSerialNumber
build.library.cyclonedx_bom_link.data.version = 1
}

/**
* Write an Concert Build Manifest a YAML file
*/
def writeBuildManifest(File yamlFile, String fileEncoding, String verbose){
println("** Generate IBM Concert Build Manifest file to '$yamlFile'")
def yamlBuilder = new YamlBuilder()

yamlBuilder {
specVersion concertManifest.specVersion
concert concertManifest.concert
}

if (verbose && verbose.toBoolean()) {
println yamlBuilder.toString()
}

// write file
yamlFile.withWriter(fileEncoding) { writer ->
writer.write(yamlBuilder.toString())
}
}

/**
* IBM Concert Manifest Classes and Helpers
*/

class ConcertBuildManifest {
String specVersion = "1.0.3"
Concertdata concert
}

class Concertdata {
ArrayList<Build> builds
}

class Build {
String component_name
String output_file
String number
String version
Library library
ArrayList<Repository> repositories
}

class Repository {
String name
String url
String branch
String commit_sha
}

class Library {
String scope
String name
String version
String filename
String url
SBOMInfo cyclonedx_bom_link
}

class SBOMInfo {
String file
SBOMdata data
}

class SBOMdata {
String serial_number
String version
}
Loading