diff --git a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy index 597140e0..5fbf2099 100644 --- a/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy +++ b/Pipeline/PackageBuildOutputs/PackageBuildOutputs.groovy @@ -51,6 +51,12 @@ import com.ibm.jzos.ZFile; * Version 8 - 2024-07 * - Reworked error management and fixed few glitches * + * Version 9 - 2024-10 + * - Added the following + * a) Fields and code to generate concert manifest linking to + * concertBuildManifestGenerator.groovy + * b) Refactoring to generate SBOM details like SerialNumber from this script + * to be passed to sbomGenerator.groovy and concertBuildManifestGenerator.groovy ************************************************************************************/ // start create & publish package @@ -58,7 +64,11 @@ import com.ibm.jzos.ZFile; def scriptDir = new File(getClass().protectionDomain.codeSource.location.path).parent @Field def wdManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/WaziDeployManifestGenerator.groovy")) @Field def sbomUtilities +@Field def concertManifestGeneratorUtilities @Field def rc = 0 +@Field def sbomSerialNumber +@Field def sbomFileName +@Field def concertBuild def startTime = new Date() @@ -91,15 +101,12 @@ Map buildOutputsMap = new HashMap scmInfo = new HashMap() -if (props.generateSBOM && props.generateSBOM.toBoolean()) { - sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) - sbomUtilities.initializeSBOM(props.sbomAuthor) -} - // iterate over all build reports to obtain build output props.buildReportOrder.each { buildReportFile -> println("** Read build report data from '${buildReportFile}'.") @@ -119,6 +126,7 @@ props.buildReportOrder.each { buildReportFile -> } if (buildInfo.size() != 0) { tarFileLabel = buildInfo[0].label + buildNumber = buildInfo[0].label } // retrieve the buildResultPropertiesRecord @@ -326,7 +334,9 @@ props.buildReportOrder.each { buildReportFile -> } // generate scmInfo for Wazi Deploy Application Manifest file - if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { + if ((props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) || + (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() )) { + if (props.buildReportOrder.size() == 1) { scmInfo.put("type", "git") gitUrl = retrieveBuildResultProperty (buildResultPropertiesRecord, "giturl") @@ -348,13 +358,30 @@ if (rc == 0) { println("** There are no build outputs found in all provided build reports. Exiting.") rc = 0 } else { - + // generate SBOM only if build outputs exist + if (props.generateSBOM && props.generateSBOM.toBoolean()) { + sbomUtilities = loadScript(new File("${scriptDir}/utilities/sbomGenerator.groovy")) + sbomSerialNumber = "url:uuid:" + UUID.randomUUID().toString() + sbomFileName = "${buildNumber}_sbom.json" + sbomUtilities.initializeSBOM(props.sbomAuthor, sbomSerialNumber) + + } // Local variables // Initialize Wazi Deploy Manifest Generator if (props.generateWaziDeployAppManifest && props.generateWaziDeployAppManifest.toBoolean()) { wdManifestGeneratorUtilities.initWaziDeployManifestGenerator(props)// Wazi Deploy Application Manifest wdManifestGeneratorUtilities.setScmInfo(scmInfo) } + + // Initialize Concert Build Manifest Generator + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + // Concert Build Manifest + + concertManifestGeneratorUtilities = loadScript(new File("${scriptDir}/utilities/concertBuildManifestGenerator.groovy")) + concertManifestGeneratorUtilities.initConcertBuildManifestGenerator() + concertBuild = concertManifestGeneratorUtilities.addBuild(props.application, props.versionName, buildNumber) + concertManifestGeneratorUtilities.addRepositoryToBuild(concertBuild, scmInfo.uri, scmInfo.branch, scmInfo.shortCommit) + } def String tarFileName = (props.tarFileName) ? props.tarFileName : "${tarFileLabel}.tar" def tarFile = "$props.workDir/${tarFileName}" @@ -456,7 +483,7 @@ if (rc == 0) { } if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { - sbomUtilities.writeSBOM("$tempLoadDir/sbom.json", props.fileEncoding) + sbomUtilities.writeSBOM("$tempLoadDir/$sbomFileName", props.fileEncoding) } @@ -465,6 +492,7 @@ if (rc == 0) { // wazideploy_manifest.yml is the default name of the manifest file wdManifestGeneratorUtilities.writeApplicationManifest(new File("$tempLoadDir/wazideploy_manifest.yml"), props.fileEncoding, props.verbose) } + if (rc == 0) { @@ -567,6 +595,36 @@ if (rc == 0) { println ("** Upload package to Artifact Repository '$url'.") artifactRepositoryHelpers.upload(url, tarFile as String, user, password, props.verbose.toBoolean(), httpClientVersion) + + // generate PackageInfo for Concert Manifest file + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + concertManifestGeneratorUtilities.addLibraryInfoTobuild(concertBuild, tarFileName, url) + } + } + + if (concertManifestGeneratorUtilities && props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean() && rc == 0) { + // concert_build_manifest.yaml is the default name of the manifest file + + if (props.generateSBOM && props.generateSBOM.toBoolean() && rc == 0) { + concertManifestGeneratorUtilities.addSBOMInfoToBuild(concertBuild, sbomFileName, sbomSerialNumber) + } + concertManifestGeneratorUtilities.writeBuildManifest(new File("$tempLoadDir/concert_build_manifest.yaml"), props.fileEncoding, props.verbose) + println("** Add concert build config yaml to tar file at ${tarFile}") + // Note: https://www.ibm.com/docs/en/zos/2.4.0?topic=scd-tar-manipulate-tar-archive-files-copy-back-up-file + // To save all attributes to be restored on z/OS and non-z/OS systems : tar -UX + def processCmd = [ + "sh", + "-c", + "tar rUXf $tarFile concert_build_manifest.yaml" + ] + + def processRC = runProcess(processCmd, tempLoadDir) + rc = Math.max(rc, processRC) + if (rc == 0) { + println("** Package '${tarFile}' successfully appended with concert manifest yaml.") + } else { + println("*! [ERROR] Error appending '${tarFile}' with concert manifest yaml rc=$rc.") + } } } } @@ -643,12 +701,15 @@ def parseInput(String[] cliArgs){ // Wazi Deploy Application Manifest generation cli.wd(longOpt:'generateWaziDeployAppManifest', 'Flag indicating to generate and add the Wazi Deploy Application Manifest file.') + // Concert Build Manifest generation + cli.ic(longOpt:'generateConcertBuildManifest', 'Flag indicating to generate and add the IBM Concert Build Manifest file.') + cli.b(longOpt:'branch', args:1, argName:'branch', 'The git branch processed by the pipeline') cli.a(longOpt:'application', args:1, argName:'application', 'The name of the application') // Artifact repository options :: cli.p(longOpt:'publish', 'Flag to indicate package upload to the provided Artifact Repository server. (Optional)') - cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)') + cli.v(longOpt:'versionName', args:1, argName:'versionName', 'Name of the version/package on the Artifact repository server. (Optional)') // Artifact repository info cli.au(longOpt:'artifactRepositoryUrl', args:1, argName:'url', 'URL to the Artifact repository server. (Optional)') @@ -705,6 +766,7 @@ def parseInput(String[] cliArgs){ // cli overrides defaults set in 'packageBuildOutputs.properties' props.generateWaziDeployAppManifest = (opts.wd) ? 'true' : props.generateWaziDeployAppManifest + props.generateConcertBuildManifest = (opts.ic) ? 'true' : props.generateConcertBuildManifest props.addExtension = (opts.ae) ? 'true' : props.addExtension props.publish = (opts.p) ? 'true' : props.publish props.generateSBOM = (opts.sbom) ? 'true' : props.generateSBOM @@ -802,10 +864,6 @@ def parseInput(String[] cliArgs){ println("*! [ERROR] Missing Artifact Repository Password property. It is required when publishing the package via ArtifactRepositoryHelpers.") rc = 2 } - if (!props.'artifactRepository.directory') { - println("*! [ERROR] Missing Artifact Repository Directory property. It is required when publishing the package via ArtifactRepositoryHelpers.") - rc = 2 - } } // assess required options to generate Wazi Deploy application manifest @@ -819,6 +877,23 @@ def parseInput(String[] cliArgs){ rc = 2 } } + + // assess required options to generate Concert Build manifest + if (props.generateConcertBuildManifest && props.generateConcertBuildManifest.toBoolean()) { + if (!props.branch) { + println("*! [ERROR] Missing branch parameter ('--branch'). It is required for generating the Concert Build Manifest file.") + rc = 2 + } + if (!props.publish) { + println("*! [ERROR] Missing publish parameter ('--publish'). It is required for generating the Concert Build Manifest file.") + rc = 2 + } + if (opts.bO || opts.boFile) { + println("*! [ERROR] conflicting parameter ('-bO or -boFile'). IBM Concert Build Manifest file is created with single builds only.") + rc = 2 + } + } + return props } diff --git a/Pipeline/PackageBuildOutputs/README.md b/Pipeline/PackageBuildOutputs/README.md index 412bedab..2cc2a686 100644 --- a/Pipeline/PackageBuildOutputs/README.md +++ b/Pipeline/PackageBuildOutputs/README.md @@ -46,6 +46,9 @@ This section provides a more detailed explanation of how the PackageBuildOutputs 6. **(Optional) Publish to Artifact Repository such as JFrog Artifactory or Sonartype Nexus** 1. Publishes the TAR file to the artifact repository based on the given configuration using the ArtifactRepositoryHelpers script. Consider a Nexus RAW, or a Artifactory Generic as the repository type. **Please note**: The ArtifactRepositoryHelpers script is updated for DBB 2.0 and requires to run on JAVA 11. The publishing can be configured to pass in the artifact repository information as well as the path within the repository `directory/[versionName|buildLabel]/tarFileName` via the cli. +7. **(Optional) Generate IBM Concert Build manifest** + 1. Based on the collected build outputs information, the IBM Concert Build Manifest file is generated and saved as concert_build_manifest.yaml. This is a feeder file to publish build information into IBM Concert. It will only be generated if both sbom and packaging options are in effect. + Notes: * The script doesn't manage the deletions of artifacts. Although they are reported in the DBB Build Reports, deletions are not handled by this script. @@ -566,9 +569,23 @@ As an example, you can invoke the SBOM generation with the following command: /usr/lpp/dbb/v2r0/bin/groovyz -cp /u/mdalbin/SBOM/cyclonedx-core-java-8.0.3.jar:/u/mdalbin/SBOM/jackson-annotations-2.16.1.jar:/u/mdalbin/SBOM/jackson-core-2.16.1.jar:/u/mdalbin/SBOM/jackson-databind-2.16.1.jar:/u/mdalbin/SBOM/jackson-dataformat-xml-2.16.1.jar:/u/mdalbin/SBOM/json-schema-validator-1.2.0.jar:/u/mdalbin/SBOM/packageurl-java-1.5.0.jar /u/mdalbin/SBOM/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputsWithSBOM.groovy --workDir /u/ado/workspace/MortgageApplication/feature/consumeRetirementCalculatorServiceImpacts/build-20240312.1/logs --tarFileName MortgageApplication.tar --addExtension -s -sa "David Gilmour " ~~~~ -By default, the SBOM file is generated in the `tempPackageDir` and named `sbom.json`. +By default, the SBOM file is generated in the `tempPackageDir` and named `_sbom.json`. This way, it is automatically packaged in the TAR file that is created by the script, ensuring the package and its content are not tampered and correctly documented. +## IBM Concert Build manifest generation + +This `PackageBuildOutputs.groovy` script is able to generate an IBM Concert Build manifest based on the information contained in the DBB Build Report and the published package information. The output is a YAML file that adheres to IBM Concert Build specification YAML format. The generation of the CycloneDX SBOM is a pre-requisite as the IBM Concert Build manifest will reference the CycloneDX SBOM file for detailed information about the build outputs. + +To enable the generation of the IBM Concert Build manifest, the `-ic/--generateConcertBuildManifest` flag must be passed. + +As an example, you can invoke the generation of an IBM Concert Build manifest with the following command: + +~~~~ +/usr/lpp/dbb/v2r0/bin/groovyz -cp /u/mdalbin/SBOM/cyclonedx-core-java-8.0.3.jar:/u/mdalbin/SBOM/jackson-annotations-2.16.1.jar:/u/mdalbin/SBOM/jackson-core-2.16.1.jar:/u/mdalbin/SBOM/jackson-databind-2.16.1.jar:/u/mdalbin/SBOM/jackson-dataformat-xml-2.16.1.jar:/u/mdalbin/SBOM/json-schema-validator-1.2.0.jar:/u/mdalbin/SBOM/packageurl-java-1.5.0.jar /u/mdalbin/SBOM/dbb/Pipeline/PackageBuildOutputs/PackageBuildOutputsWithSBOM.groovy --workDir /u/ado/workspace/MortgageApplication/feature/consumeRetirementCalculatorServiceImpacts/build-20240312.1/logs --tarFileName MortgageApplication.tar --addExtension -s -sa "David Gilmour " -ic +~~~~ + +By default, the IBM Concert Build manifest file is generated in the `tempPackageDir` and named `concert_build_manifest.yaml`. + ## Useful reference material diff --git a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties index 999eea13..094467b6 100644 --- a/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties +++ b/Pipeline/PackageBuildOutputs/packageBuildOutputs.properties @@ -30,6 +30,11 @@ addExtension=false # Default: false generateWaziDeployAppManifest=false +# Boolean setting to define if the IBM Concert Build Manifest file should be generated +# Please note that the cli option `generateConcertBuildManifest` can override this setting and activate it. +# Default: false +generateConcertBuildManifest=false + # Boolean setting to define if SBOM based on cycloneDX should be generated # Please note that the cli option `generateSBOM` can override this setting and activate it. # Requires the classpath to contain the cycloneDX libs diff --git a/Pipeline/PackageBuildOutputs/utilities/concertBuildManifestGenerator.groovy b/Pipeline/PackageBuildOutputs/utilities/concertBuildManifestGenerator.groovy new file mode 100644 index 00000000..1ab3fd3f --- /dev/null +++ b/Pipeline/PackageBuildOutputs/utilities/concertBuildManifestGenerator.groovy @@ -0,0 +1,123 @@ +import groovy.transform.* +import groovy.yaml.YamlBuilder +import com.ibm.dbb.build.report.records.* + +/* + * This is a utility method to generate IBM Concert build files + */ + +@Field ConcertBuildManifest concertManifest = new ConcertBuildManifest() + +def initConcertBuildManifestGenerator() { + // Metadata + concertManifest.concert = new Concertdata() + concertManifest.concert.builds = new ArrayList() +} + +def addBuild(String application, String version, String buildNumber) { + Build build = new Build() + build.repositories = new ArrayList() + build.library = new Library() + build.component_name = application + build.number = buildNumber + build.output_file = application + "_sbom.json" + // Metadata information + build.version = version + concertManifest.concert.builds.add(build) + return build +} + +def addRepositoryToBuild(Build build, String url, String branch, String shortCommit) { + Repository repository = new Repository() + repository.name = build.component_name + repository.url = url + repository.branch = branch + repository.commit_sha = shortCommit + build.repositories.add(repository) +} + +def addLibraryInfoTobuild(Build build, String filename, String url) { + build.library.scope = 'tar' + build.library.name = build.component_name + build.library.filename = filename + build.library.version = build.version + build.library.url = url +} + +def addSBOMInfoToBuild(Build build, String sbomFileName, String sbomSerialNumber) { + build.library.cyclonedx_bom_link = new SBOMInfo() + build.library.cyclonedx_bom_link.file = sbomFileName + build.library.cyclonedx_bom_link.data = new SBOMdata() + build.library.cyclonedx_bom_link.data.serial_number = sbomSerialNumber + build.library.cyclonedx_bom_link.data.version = 1 +} + +/** + * Write an Concert Build Manifest a YAML file + */ +def writeBuildManifest(File yamlFile, String fileEncoding, String verbose){ + println("** Generate IBM Concert Build Manifest file to '$yamlFile'") + def yamlBuilder = new YamlBuilder() + + yamlBuilder { + specVersion concertManifest.specVersion + concert concertManifest.concert + } + + if (verbose && verbose.toBoolean()) { + println yamlBuilder.toString() + } + + // write file + yamlFile.withWriter(fileEncoding) { writer -> + writer.write(yamlBuilder.toString()) + } +} + +/** + * IBM Concert Manifest Classes and Helpers + */ + +class ConcertBuildManifest { + String specVersion = "1.0.3" + Concertdata concert +} + +class Concertdata { + ArrayList builds +} + +class Build { + String component_name + String output_file + String number + String version + Library library + ArrayList repositories +} + +class Repository { + String name + String url + String branch + String commit_sha +} + +class Library { + String scope + String name + String version + String filename + String url + SBOMInfo cyclonedx_bom_link +} + +class SBOMInfo { + String file + SBOMdata data +} + +class SBOMdata { + String serial_number + String version +} \ No newline at end of file diff --git a/Pipeline/PackageBuildOutputs/utilities/sbomGenerator.groovy b/Pipeline/PackageBuildOutputs/utilities/sbomGenerator.groovy index aa19b6b9..6bfd154e 100644 --- a/Pipeline/PackageBuildOutputs/utilities/sbomGenerator.groovy +++ b/Pipeline/PackageBuildOutputs/utilities/sbomGenerator.groovy @@ -19,6 +19,9 @@ import org.cyclonedx.model.* * * Version 1 - 04/2024 * Initial implementation of SBOM Generation + * Version 2 - 10/2024 + * Changed to pass SerialNumber from the packaging script. Changed type "container" + * to "library" ************************************************************************************/ @@ -30,10 +33,8 @@ import org.cyclonedx.model.* @Field ArrayList sbomDependencies @Field Bom sbom -def initializeSBOM(String sbomAuthor) { +def initializeSBOM(String sbomAuthor, String serialNumber) { sbom = new Bom(); - sbom.setSerialNumber("url:uuid:" + UUID.randomUUID().toString()); - sbom.setVersion(1); LifecycleChoice sbomLifecycleChoice = new LifecycleChoice() sbomLifecycleChoice.setPhase(LifecycleChoice.Phase.POST_BUILD) Lifecycles sbomLifecycles = new Lifecycles() @@ -49,11 +50,19 @@ def initializeSBOM(String sbomAuthor) { author.setEmail(sbomAuthorFields[1].replaceAll(">", "").trim()) sbomMetadata.addAuthor(author) } else { - println("*! Warning: SBOM Author not correctly formed, expecting 'Name ' format. Skipping.") + println("*! [WARNING] SBOM Author not correctly formed, expecting 'Name ' format. Skipping.") } } else { - println("*! Warning: empty SBOM Author. It is recommend to specify a valid Author.") + println("*! [WARNING] empty SBOM Author. It is recommend to specify a valid Author.") } + if (serialNumber) { + sbom.setSerialNumber(serialNumber) + } else { + println("*! [WARNING] Serial Number has been regenerated.") + sbom.setSerialNumber("url:uuid:" + UUID.randomUUID().toString()) + } + + sbom.setVersion(1) sbom.setMetadata(sbomMetadata) sbom.setDependencies(new ArrayList()) } @@ -95,7 +104,7 @@ def addEntryToSBOM(DeployableArtifact deployableArtifact, HashMap deployableArtifactComponentProperties = new ArrayList() Property deployableArtifactContainerComponentProperty = new Property() - deployableArtifactContainerComponentProperty.setName("container") + deployableArtifactContainerComponentProperty.setName("library") deployableArtifactContainerComponentProperty.setValue(container) deployableArtifactComponentProperties.add(deployableArtifactContainerComponentProperty) Property deployableArtifactDeployTypeComponentProperty = new Property()