-
Notifications
You must be signed in to change notification settings - Fork 149
Feature/concert SBOM integration #283
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
2abbf34
60ec298
d5a05bd
eef5e0b
7afa3dd
94a4e11
93c8e5e
11afdd8
e83fb34
c007293
f155e0b
ec10f44
c46bd2e
8312093
e66e03b
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -51,14 +51,24 @@ import com.ibm.jzos.ZFile; | |
| * Version 8 - 2024-07 | ||
| * - Reworked error management and fixed few glitches | ||
| * | ||
| * Version 9 - 2024-10 | ||
| * - Added the following | ||
| * a) Fields and code to generate concert manifest linking to | ||
| * concertBuildManifestGenerator.groovy | ||
| * b) Refactoring to generate SBOM details like SerialNumber from this script | ||
| * to be passed to sbomGenerator.groovy and concertBuildManifestGenerator.groovy | ||
| ************************************************************************************/ | ||
|
|
||
| // start create & publish package | ||
| @Field Properties props = null | ||
| def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent | ||
| @Field def wdManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/WaziDeployManifestGenerator.groovy")) | ||
| @Field def sbomUtilities | ||
| @Field def concertManifestGeneratorUtilities | ||
| @Field def rc = 0 | ||
| @Field def sbomSerialNumber | ||
| @Field def sbomFileName | ||
| @Field def concertBuild | ||
|
|
||
| def startTime = new Date() | ||
|
|
||
|
|
@@ -91,15 +101,12 @@ Map<DeployableArtifact, Map> buildOutputsMap = new HashMap<DeployableArtifact, M | |
|
|
||
| // Field to store default tarFileLabel (buildInfo.label) when cli argument tarFileName is not passed. | ||
| def String tarFileLabel = "Default" | ||
| // Field to store build number, set to default when its not decipherable from build report | ||
| def String buildNumber = "UNKNOWN" | ||
|
|
||
| // Object to store scm information for Wazi Deploy Application Manifest file | ||
| HashMap<String,String> scmInfo = new HashMap<String, String>() | ||
|
|
||
| if (props.generateSBOM && props.generateSBOM.toBoolean()) { | ||
| sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) | ||
| sbomUtilities.initializeSBOM(props.sbomAuthor) | ||
| } | ||
|
|
||
| // iterate over all build reports to obtain build output | ||
| props.buildReportOrder.each { buildReportFile -> | ||
| println("** Read build report data from '${buildReportFile}'.") | ||
|
|
@@ -119,6 +126,7 @@ props.buildReportOrder.each { buildReportFile -> | |
| } | ||
| if (buildInfo.size() != 0) { | ||
| tarFileLabel = buildInfo[0].label | ||
| buildNumber = buildInfo[0].label | ||
| } | ||
|
|
||
| // retrieve the buildResultPropertiesRecord | ||
|
|
@@ -326,7 +334,9 @@ props.buildReportOrder.each { buildReportFile -> | |
| } | ||
|
|
||
| // generate scmInfo for Wazi Deploy Application Manifest file | ||
| if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { | ||
| if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || | ||
| (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { | ||
|
|
||
| if (props.buildReportOrder.size() == 1) { | ||
| scmInfo.put("type", "git") | ||
| gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl") | ||
|
|
@@ -348,13 +358,30 @@ if (rc == 0) { | |
| println("** There are no build outputs found in all provided build reports. Exiting.") | ||
| rc = 0 | ||
| } else { | ||
|
|
||
| // generate SBOM only if build outputs exist | ||
| if (props.generateSBOM && props.generateSBOM.toBoolean()) { | ||
| sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) | ||
| sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString() | ||
| sbomFileName = "${buildNumber}_sbom.json" | ||
| sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber) | ||
|
|
||
| } | ||
| // Local variables | ||
| // Initialize Wazi Deploy Manifest Generator | ||
| if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { | ||
| wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest | ||
| wdManifestGeneratorUtilities.setScmInfo(scmInfo) | ||
| } | ||
|
|
||
| // Initialize Concert Build Manifest Generator | ||
| if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { | ||
| // Concert Build Manifest | ||
|
|
||
| concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy")) | ||
| concertManifestGeneratorUtilities.initConcertBuildManifestGenerator() | ||
| concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber) | ||
| concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit) | ||
| } | ||
| def String tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" | ||
| def tarFile = "$props.workDir/${tarFileName}" | ||
|
|
||
|
|
@@ -456,7 +483,7 @@ if (rc == 0) { | |
| } | ||
|
|
||
| if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { | ||
| sbomUtilities.writeSBOM("$tempLoadDir/sbom.json", props.fileEncoding) | ||
| sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) | ||
| } | ||
|
|
||
|
|
||
|
|
@@ -465,6 +492,7 @@ if (rc == 0) { | |
| // wazideploy_manifest.yml is the default name of the manifest file | ||
| wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) | ||
| } | ||
|
|
||
|
|
||
| if (rc == 0) { | ||
|
|
||
|
|
@@ -567,6 +595,36 @@ if (rc == 0) { | |
|
|
||
| println ("** Upload package to Artifact Repository '$url'.") | ||
| artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) | ||
|
|
||
| // generate PackageInfo for Concert Manifest file | ||
| if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { | ||
| concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url) | ||
| } | ||
| } | ||
|
|
||
| if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) { | ||
| // concert_build_manifest.yaml is the default name of the manifest file | ||
|
|
||
| if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { | ||
| concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber) | ||
| } | ||
| concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose) | ||
| println("** Add concert build config yaml to tar file at ${tarFile}") | ||
| // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file | ||
| // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX | ||
| def processCmd = [ | ||
| "sh", | ||
| "-c", | ||
| "tar rUXf $tarFile concert_build_manifest.yaml" | ||
| ] | ||
|
|
||
| def processRC = runProcess(processCmd, tempLoadDir) | ||
| rc = Math.max(rc, processRC) | ||
| if (rc == 0) { | ||
| println("** Package '${tarFile}' successfully appended with concert manifest yaml.") | ||
| } else { | ||
| println("*! [ERROR] Error appending '${tarFile}' with concert manifest yaml rc=$rc.") | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
@@ -643,12 +701,15 @@ def parseInput(String[] cliArgs){ | |
| // Wazi Deploy Application Manifest generation | ||
| cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.') | ||
|
|
||
| // Concert Build Manifest generation | ||
| cli.ic(longOpt:'generateConcertBuildManifest', 'Flag indicating to generate and add the IBM Concert Build Manifest file.') | ||
|
|
||
| cli.b(longOpt:'branch', args:1, argName:'branch', 'The git branch processed by the pipeline') | ||
| cli.a(longOpt:'application', args:1, argName:'application', 'The name of the application') | ||
|
|
||
| // Artifact repository options :: | ||
| cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifact Repository server. (Optional)') | ||
| cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)') | ||
| cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)') | ||
|
|
||
| // Artifact repository info | ||
| cli.au(longOpt:'artifactRepositoryUrl', args:1, argName:'url', 'URL to the Artifact repository server. (Optional)') | ||
|
|
@@ -705,6 +766,7 @@ def parseInput(String[] cliArgs){ | |
|
|
||
| // cli overrides defaults set in 'packageBuildOutputs.properties' | ||
| props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : props.generateWaziDeployAppManifest | ||
| props.generateConcertBuildManifest = (opts.ic) ? 'true' : props.generateConcertBuildManifest | ||
| props.addExtension = (opts.ae) ? 'true' : props.addExtension | ||
| props.publish = (opts.p) ? 'true' : props.publish | ||
| props.generateSBOM = (opts.sbom) ? 'true' : props.generateSBOM | ||
|
|
@@ -802,10 +864,6 @@ def parseInput(String[] cliArgs){ | |
| println("*! [ERROR] Missing Artifact Repository Password property. It is required when publishing the package via ArtifactRepositoryHelpers.") | ||
| rc = 2 | ||
| } | ||
| if (!props.'artifactRepository.directory') { | ||
| println("*! [ERROR] Missing Artifact Repository Directory property. It is required when publishing the package via ArtifactRepositoryHelpers.") | ||
| rc = 2 | ||
| } | ||
| } | ||
|
|
||
| // assess required options to generate Wazi Deploy application manifest | ||
|
|
@@ -819,6 +877,23 @@ def parseInput(String[] cliArgs){ | |
| rc = 2 | ||
| } | ||
| } | ||
|
|
||
| // assess required options to generate Concert Build manifest | ||
| if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { | ||
| if (!props.branch) { | ||
| println("*! [ERROR] Missing branch parameter ('--branch'). It is required for generating the Concert Build Manifest file.") | ||
| rc = 2 | ||
| } | ||
| if (!props.publish) { | ||
| println("*! [ERROR] Missing publish parameter ('--publish'). It is required for generating the Concert Build Manifest file.") | ||
| rc = 2 | ||
| } | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it a correct assumption that generating the concert build manifest file, will only work when only one BuildReport.json is passed to the PackageBuildOutputsScript? If so, we should validate that in this block.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We will need to discuss this
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. done. Added the check. Need some more tests |
||
| if (opts.bO || opts.boFile) { | ||
| println("*! [ERROR] conflicting parameter ('-bO or -boFile'). IBM Concert Build Manifest file is created with single builds only.") | ||
| rc = 2 | ||
| } | ||
| } | ||
|
|
||
| return props | ||
| } | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| import groovy.transform.* | ||
| import groovy.yaml.YamlBuilder | ||
| import com.ibm.dbb.build.report.records.* | ||
|
|
||
| /* | ||
| * This is a utility method to generate IBM Concert build files | ||
| */ | ||
|
|
||
| @Field ConcertBuildManifest concertManifest = new ConcertBuildManifest() | ||
|
|
||
| def initConcertBuildManifestGenerator() { | ||
| // Metadata | ||
| concertManifest.concert = new Concertdata() | ||
| concertManifest.concert.builds = new ArrayList<Build>() | ||
| } | ||
|
|
||
| def addBuild(String application, String version, String buildNumber) { | ||
| Build build = new Build() | ||
| build.repositories = new ArrayList<Repository>() | ||
| build.library = new Library() | ||
| build.component_name = application | ||
| build.number = buildNumber | ||
| build.output_file = application + "_sbom.json" | ||
| // Metadata information | ||
| build.version = version | ||
| concertManifest.concert.builds.add(build) | ||
| return build | ||
| } | ||
|
|
||
| def addRepositoryToBuild(Build build, String url, String branch, String shortCommit) { | ||
| Repository repository = new Repository() | ||
| repository.name = build.component_name | ||
| repository.url = url | ||
| repository.branch = branch | ||
| repository.commit_sha = shortCommit | ||
| build.repositories.add(repository) | ||
| } | ||
|
|
||
| def addLibraryInfoTobuild(Build build, String filename, String url) { | ||
| build.library.scope = 'tar' | ||
| build.library.name = build.component_name | ||
| build.library.filename = filename | ||
| build.library.version = build.version | ||
| build.library.url = url | ||
| } | ||
|
|
||
| def addSBOMInfoToBuild(Build build, String sbomFileName, String sbomSerialNumber) { | ||
| build.library.cyclonedx_bom_link = new SBOMInfo() | ||
| build.library.cyclonedx_bom_link.file = sbomFileName | ||
| build.library.cyclonedx_bom_link.data = new SBOMdata() | ||
| build.library.cyclonedx_bom_link.data.serial_number = sbomSerialNumber | ||
| build.library.cyclonedx_bom_link.data.version = 1 | ||
| } | ||
|
|
||
| /** | ||
| * Write an Concert Build Manifest a YAML file | ||
| */ | ||
| def writeBuildManifest(File yamlFile, String fileEncoding, String verbose){ | ||
| println("** Generate IBM Concert Build Manifest file to '$yamlFile'") | ||
| def yamlBuilder = new YamlBuilder() | ||
|
|
||
| yamlBuilder { | ||
| specVersion concertManifest.specVersion | ||
| concert concertManifest.concert | ||
| } | ||
|
|
||
| if (verbose && verbose.toBoolean()) { | ||
| println yamlBuilder.toString() | ||
| } | ||
|
|
||
| // write file | ||
| yamlFile.withWriter(fileEncoding) { writer -> | ||
| writer.write(yamlBuilder.toString()) | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * IBM Concert Manifest Classes and Helpers | ||
| */ | ||
|
|
||
| class ConcertBuildManifest { | ||
| String specVersion = "1.0.3" | ||
| Concertdata concert | ||
| } | ||
|
|
||
| class Concertdata { | ||
| ArrayList<Build> builds | ||
| } | ||
|
|
||
| class Build { | ||
| String component_name | ||
| String output_file | ||
| String number | ||
| String version | ||
| Library library | ||
| ArrayList<Repository> repositories | ||
| } | ||
|
|
||
| class Repository { | ||
| String name | ||
| String url | ||
| String branch | ||
| String commit_sha | ||
| } | ||
|
|
||
| class Library { | ||
| String scope | ||
| String name | ||
| String version | ||
| String filename | ||
| String url | ||
| SBOMInfo cyclonedx_bom_link | ||
| } | ||
|
|
||
| class SBOMInfo { | ||
| String file | ||
| SBOMdata data | ||
| } | ||
|
|
||
| class SBOMdata { | ||
| String serial_number | ||
| String version | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is the file name format rather something that we should declare at the beginning of the script?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It looks like the buildNumber is retrieved from the build report. So, it might make sense to retain this where it is.