Merge pull request #430 from IBM/develop

Release zAppBuild 3.5.0

build-conf/PLI.properties

@@ -2,7 +2,7 @@
 
 #
 # Comma separated list of required build properties for PLI.groovy
-pli_requiredBuildPropeties=pli_srcPDS,pli_incPDS,pli_objPDS,pli_loadPDS,\
+pli_requiredBuildProperties=pli_srcPDS,pli_incPDS,pli_objPDS,pli_loadPDS,\
  pli_compiler,pli_linkEditor,pli_tempOptions,applicationOutputsCollectionName,\
  SDFHCOB,SDFHLOAD,SDSNLOAD,SCEELKED, \
  pli_dependencySearch

build-conf/README.md

@@ -28,6 +28,7 @@ SCSQCOBC | MQ COBOL Library. Example: CSQ.V9R1M0.SCSQCOBC
 SCSQPLIC | MQ PLI Library. Example: CSQ.V9R1M0.SCSQPLIC
 SCSQLOAD | MQ Load Library. Example: CSQ.V9R1M0.SCSQLOAD
 SDSNLOAD | DB2 Load Library. Example: DB2.V9R1M0.SDSNLOAD
+SDSNEXIT | DB2 Exit Library. Example: DBC0CFG.SDSNEXIT
 SFELLOAD | Optional IDz Load Library. Example: FEL.V14R0M0.SFELLOAD
 SBZUSAMP | Optional z/OS Dynamic Test Runner IDz zUnit / WAZI VTP library containing necessary copybooks. Example : FEL.V14R2.SBZUSAMP
 
@@ -41,7 +42,6 @@ applicationDefaultPropFiles | Comma separated list of default application confi
 buildListFileExt | File extension that indicates the build file is really a build list.
 applicationConfRootDir | Alternate root directory for application-conf location.  Allows for the deployment of the application-conf directories to a static location.  Defaults to ${workspace}/${application}
 createBuildOutputSubfolder | Option to create a subfolder with the build label within the build output dir (outDir). Default: true.
-buildOutputTSformat | Defines the build timestamp format for build output subfolder and build label.
 requiredDBBToolkitVersion | Minimum required DBB ToolkitVersion to run this version of zAppBuild.
 requiredBuildProperties | Comma separated list of required build properties for zAppBuild/build.groovy. Build and language scripts will validate that *required* build properties have been set before the script runs.  If any are missing or empty, then a validation error will be thrown.
 dbb.file.tagging | Controls compile log and build report file tagging. Default: true.
@@ -323,8 +323,8 @@ transfer_requiredBuildProperties | Comma separated list of required build proper
 transfer_srcPDS | Dataset of any type of source
 transfer_jclPDS | Sample dataset for JCL members
 transfer_xmlPDS | Sample dataset for xml members
-transfer_srcOptions | BPXWDYN creation options for creating 'source' type data sets
-transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets 
+transfer_dsOptions | BPXWDYN creation options for creating 'source' type data sets
+transfer_outputDatasets | List of output datasets to document deletions ** Can be overridden by a file property. ** If used for multiple, use a file property to set transfer_outputDatasets
 
 ### language-conf/languageConfigProps01.properties
 Sample language configuration properties file used by dbb-zappbuild/utilities/BuildUtilities.groovy.

build-conf/Transfer.properties

@@ -20,6 +20,6 @@ transfer_xmlPDS=${hlq}.XML
 #
 # This is using the DBB PropertyMapping syntax allowing multiple mappings of
 #  target source dataset definitions to the required options
-# 
+#
 transfer_dsOptions=cyl space(1,1) lrecl(80) dsorg(PO) recfm(F,B) dsntype(library) :: transfer_srcPDS, transfer_jclPDS
-transfer_dsOptions=cyl space(1,1) lrecl(144) dsorg(PO) recfm(F,B) dsntype(library) :: transfer_xmlPDS
+transfer_dsOptions=cyl space(1,1) lrecl(144) dsorg(PO) recfm(F,B) dsntype(library) :: transfer_xmlPDS

build-conf/build.properties

@@ -14,7 +14,7 @@
 # Supports both relative path (to zAppBuild/build-conf/) and absolute path
 #
 # These properties files expect to contain centrally managed defaults
-#  such as system datasets, language script specific settings  
+#  such as system datasets, language script specific settings
 #
 buildPropFiles=datasets.properties,dependencyReport.properties,Assembler.properties,BMS.properties,\
 MFS.properties,PSBgen.properties,DBDgen.properties,ACBgen.properties,Cobol.properties,\
@@ -24,14 +24,14 @@ CRB.properties,zCEE3.properties
 #
 # Comma separated list of default application configuration property files to load
 # Supports both relative path (to zAppBuild/build-conf/) and absolute path
-# 
+#
 # These properties files expect to contain centrally managed defaults
 #  and also may reference properties files containing configuration
-#  of the language script configurations such as return codes, deploy types   
-# 
+#  of the language script configurations such as return codes, deploy types
+#
 # See also application-conf/application.properties#applicationPropFiles
-# 
-# default: 
+#
+# default:
 # applicationDefaultPropFiles=defaultzAppBuildConf.properties
 #
 # extended sample to set default language script configurations:
@@ -45,7 +45,7 @@ CRB.properties,zCEE3.properties
 # default-application-conf/Transfer.properties,\
 # default-application-conf/LinkEdit.properties,\
 # default-application-conf/ZunitConfig.properties
-# 
+#
 applicationDefaultPropFiles=defaultzAppBuildConf.properties
 
 #
@@ -58,7 +58,7 @@ applicationDefaultPropFiles=defaultzAppBuildConf.properties
 # a sample of the application-conf can be found in
 #   samples/application-conf
 #
-# zAppBuild expects a file called application.properties in this directory. 
+# zAppBuild expects a file called application.properties in this directory.
 #
 # The property also allows for the deployment of
 # the application-conf directories to an alternate location rather
@@ -69,7 +69,7 @@ applicationDefaultPropFiles=defaultzAppBuildConf.properties
 # based on workspace + application and application-conf
 #
 # applicationConfDir=${workspace}/${application}/application-conf
-# 
+#
 #
 # Example: Static location on USS
 # applicationConfDir=/u/build/config/applications/${application}/application-conf
@@ -101,12 +101,6 @@ buildListFileExt=txt
 # Default: true
 createBuildOutputSubfolder=true
 
-#
-# Build Timestamp Format
-# Applies to all build types except userBuild
-# Default: yyyyMMdd.HHmmss.mmm - See Date format method pattern strings
-buildOutputTSformat=yyyyMMdd.HHmmss.mmm
-
 #
 # Minimum required DBB ToolkitVersion to run this version of zAppBuild
 #  Build initialization process validates the DBB Toolkit Version in use and matches that against this setting

build-conf/datasets.properties

@@ -51,6 +51,9 @@ SCSQLOAD=
 # DB2 Load Library. Example: DB2.V9R1M0.SDSNLOAD
 SDSNLOAD=
 
+# DB2 Exit Library. Example: DBC0CFG.SDSNEXIT
+SDSNEXIT=
+
 # IMS Macro Library. Example: DFS.V11R1M0.SDFSMAC
 SDFSMAC=
 

build.groovy

@@ -27,7 +27,7 @@ import groovy.cli.commons.*
 @Field startTime = new Date()
 
 // start time message
-props.startTime = startTime.format("yyyyMMdd.hhmmss.mmm")
+props.startTime = startTime.format("yyyyMMdd.HHmmss.SSS")
 println("\n** Build start at $props.startTime")
 
 // initialize build
@@ -89,7 +89,7 @@ if (props.error)
 //********************************************************************
 
 def initializeBuildProcess(String[] args) {
-	if (props.verbose) println "** Initializing build process . . ."
+    println "**************** Initialization of the build process ****************"
 
 	def opts = parseArgs(args) // parse incoming options and arguments
 	populateBuildProperties(opts) // build properties initial set
@@ -294,8 +294,8 @@ options:
 		System.exit(1)
 	}
 
-	if(opts.v && args.size() > 1)
-		println "** Input args = ${args[1..-1].join(' ')}"
+	if(opts.v && args.size() > 0)
+		println "** Input args = ${args[0..-1].join(' ')}"
 
 	if( (!opts.cch && opts.ccp) || (opts.cch && !opts.ccp) ) {
 		println "** --cccHost and --cccPort options are mutual"
@@ -490,7 +490,7 @@ def populateBuildProperties(def opts) {
 
 	props.topicBranchBuild = (props.applicationCurrentBranch.equals(props.mainBuildBranch)) ? null : 'true'
 	props.applicationBuildGroup = ((props.applicationCurrentBranch) ? "${props.application}-${props.applicationCurrentBranch}" : "${props.application}") as String
-	props.applicationBuildLabel = ("build." + ( (props.buildOutputTSformat) ? startTime.format("${props.buildOutputTSformat}") : "${props.startTime}" ) ) as String
+	props.applicationBuildLabel = ("build.${props.startTime}") as String
 	props.applicationCollectionName = ((props.applicationCurrentBranch) ? "${props.application}-${props.applicationCurrentBranch}" : "${props.application}") as String
 	props.applicationOutputsCollectionName = "${props.applicationCollectionName}-outputs" as String
 
@@ -526,6 +526,7 @@ def populateBuildProperties(def opts) {
  *   - build text file: Contains a list of programs from a text file. Provide a *.txt build file argument.
  */
 def createBuildList() {
+    println "************* Creation and processing of the build list *************"
 
 	// using a set to create build list to eliminate duplicate files
 	Set<String> buildSet = new HashSet<String>()
@@ -671,7 +672,7 @@ def createBuildList() {
 
 
 def finalizeBuildProcess(Map args) {
-
+    println "***************** Finalization of the build process *****************"
 
 	def buildReport = BuildReportFactory.getBuildReport()
 	def buildResult = null

languages/Assembler.groovy

@@ -213,8 +213,14 @@ def createAssemblerSQLTranslatorCommand(String buildFile, LogicalFile logicalFil
 	assembler_SQLtranslator.dd(new DDStatement().name("SYSCIN").dsn("&&SYSCIN").options('cyl space(5,5) unit(vio) new').pass(true))
 
 	// steplib
-	assembler_SQLtranslator.dd(new DDStatement().name("TASKLIB").dsn(props.SDSNLOAD).options("shr"))
+	if (props.SDSNEXIT) {
+		assembler_SQLtranslator.dd(new DDStatement().name("TASKLIB").dsn(props.SDSNEXIT).options("shr"))
+		assembler_SQLtranslator.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr"))
 
+	} else {
+		assembler_SQLtranslator.dd(new DDStatement().name("TASKLIB").dsn(props.SDSNLOAD).options("shr"))
+	}
+
 	assembler_SQLtranslator.dd(new DDStatement().name("SYSUT1").options(props.assembler_tempOptions))
 
 	// sysprint

languages/Cobol.groovy

@@ -261,8 +261,10 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe
 	compile.dd(new DDStatement().name("TASKLIB").dsn(props."SIGYCOMP_$compilerVer").options("shr"))
 	if (buildUtils.isCICS(logicalFile))
 		compile.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr"))
-	if (buildUtils.isSQL(logicalFile))
+	if (buildUtils.isSQL(logicalFile)) {
+		if (props.SDSNEXIT) compile.dd(new DDStatement().dsn(props.SDSNEXIT).options("shr"))
 		compile.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr"))
+	}
 
 	if (props.SFELLOAD)
 		compile.dd(new DDStatement().dsn(props.SFELLOAD).options("shr"))

languages/MFS.groovy

@@ -4,7 +4,7 @@ import com.ibm.dbb.dependency.*
 import com.ibm.dbb.build.*
 import groovy.transform.*
 
-// docs: https://www.ibm.com/support/knowledgecenter/en/SSEPH2_13.1.0/com.ibm.ims13.doc.sur/ims_mfslangbatchmd.htm
+// docs: https://www.ibm.com/docs/en/ims/14.1.0?topic=dfsupaa0-running-utility-in-standard-mode
 
 // define script properties
 @Field BuildProperties props = BuildProperties.getInstance()
@@ -34,14 +34,17 @@ sortedList.each { buildFile ->
 	File logFile = new File( props.userBuild ? "${props.buildOutDir}/${member}.log" : "${props.buildOutDir}/${member}.mfs.log")
 	if (logFile.exists())
 		logFile.delete()
-	MVSExec phase1 = createPhase1Command(buildFile, member, logFile)
-	MVSExec phase2 = createPhase2Command(buildFile, member, logFile)
-
+
+	// execution flags
+	phase2Execution = props.getFileProperty('mfs_phase2Execution', buildFile)
+
 	// execute mvs commands in a mvs job
 	MVSJob job = new MVSJob()
 	job.start()
 
-	// preprocess mfs map
+	// generate phase1 command
+	MVSExec phase1 = createPhase1Command(buildFile, member, logFile)
+
 	int rc = phase1.execute()
 	int maxRC = props.getFileProperty('mfs_phase1MaxRC', buildFile).toInteger()
 
@@ -53,19 +56,24 @@ sortedList.each { buildFile ->
 		buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile])
 	}
 	else {
+		// generate phase2 command
+		if (phase2Execution && phase2Execution.toBoolean()) {
+
+			MVSExec phase2 = createPhase2Command(buildFile, member, logFile)
+
+			rc = phase2.execute()
+			maxRC = props.getFileProperty('mfs_phase2MaxRC', buildFile).toInteger()
 
-		rc = phase2.execute()
-		maxRC = props.getFileProperty('mfs_phase2MaxRC', buildFile).toInteger()
+			if (rc > maxRC) {
+				String errorMsg = "*! The phase 2 return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)"
+				println(errorMsg)
+				props.error = "true"
+				buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile])
+			}
 
-		if (rc > maxRC) {
-			String errorMsg = "*! The phase 2 return code ($rc) for $buildFile exceeded the maximum return code allowed ($maxRC)"
-			println(errorMsg)
-			props.error = "true"
-			buildUtils.updateBuildResult(errorMsg:errorMsg,logs:["${member}.log":logFile])
 		}
 	}
 
-
 	// clean up passed DD statements
 	job.stop()
 
@@ -81,6 +89,9 @@ sortedList.each { buildFile ->
 
 /*
  * createPhase1Command - creates a MVSExec command for preprocessing the MFS Map (buildFile)
+ * 
+ * defines the MFS map as output.
+ * 
  */
 def createPhase1Command(String buildFile, String member, File logFile) {
 
@@ -91,11 +102,15 @@ def createPhase1Command(String buildFile, String member, File logFile) {
 	// define the MVSExec command to compile the mfs map
 	MVSExec mfsPhase1 = new MVSExec().file(buildFile).pgm(props.mfs_phase1processor).parm(parameters)
 
-	mfsPhase1.dd(new DDStatement().name("SYSIN").dsn("${props.mfs_srcPDS}($member)").options("shr").report(true))
+	// add DD statements to the mfsPhase2 command
+	String deployType = buildUtils.getDeployType("mfs", buildFile, null)
+
+	mfsPhase1.dd(new DDStatement().name("SYSIN").dsn("${props.mfs_srcPDS}($member)").options("shr").report(true).output(true).deployType(deployType))
 
-	mfsPhase1.dd(new DDStatement().name("REFIN").dsn(props.REFERAL).options("shr"))
+	mfsPhase1.dd(new DDStatement().name("REFIN").dsn("&&REFERAL").options("${props.mfs_tempOptions} dir(10) lrecl(80) blksize(800) recfm(f,b)"))
 	mfsPhase1.dd(new DDStatement().name("REFOUT").dsn("&&TEMPPDS").options("${props.mfs_tempOptions} dir(5) lrecl(80) recfm(f,b)"))
-	mfsPhase1.dd(new DDStatement().name("REFRD").dsn(props.REFERAL).options("shr"))
+	mfsPhase1.dd(new DDStatement().name("REFRD").dsn("&&TEMPPDS").options("cyl space(5,5) unit(vio) old"))
+	mfsPhase1.dd(new DDStatement().dsn("&&REFERAL").options("cyl space(5,5) unit(vio) old"))
 
 	mfsPhase1.dd(new DDStatement().name("SYSPRINT").options(props.mfs_tempOptions))
 	mfsPhase1.dd(new DDStatement().name("SEQBLKS").dsn("&&SEQBLK").options(props.mfs_tempOptions).pass(true))
@@ -128,10 +143,7 @@ def createPhase2Command(String buildFile, String member, File logFile) {
 	// define the MVSExec command for MFS Language Utility - Phase 2
 	MVSExec mfsPhase2 = new MVSExec().file(buildFile).pgm(props.mfs_phase2processor).parm(parameters)
 
-	// add DD statements to the mfsPhase2 command
-	String deployType = buildUtils.getDeployType("mfs", buildFile, null)
-
-	mfsPhase2.dd(new DDStatement().name("FORMAT").dsn(props.mfs_tformatPDS).options("shr").output(true).deployType(deployType))
+	mfsPhase2.dd(new DDStatement().name("FORMAT").dsn(props.mfs_tformatPDS).options("shr"))
 	// mfsPhase2.dd(new DDStatement().name("DUMMY").dsn("${props.PROCLIB}(FMTCPY)").options("shr"))
 	mfsPhase2.dd(new DDStatement().name("TASKLIB").dsn(props.SDFSRESL).options("shr"))
 

languages/PLI.groovy

@@ -222,8 +222,10 @@ def createCompileCommand(String buildFile, LogicalFile logicalFile, String membe
 	compile.dd(new DDStatement().name("TASKLIB").dsn(props."IBMZPLI_$compilerVer").options("shr"))
 	if (buildUtils.isCICS(logicalFile))
 		compile.dd(new DDStatement().dsn(props.SDFHLOAD).options("shr"))
-	if (buildUtils.isSQL(logicalFile))
+	if (buildUtils.isSQL(logicalFile)) {
+		if (props.SDSNEXIT) compile.dd(new DDStatement().dsn(props.SDSNEXIT).options("shr"))
 		compile.dd(new DDStatement().dsn(props.SDSNLOAD).options("shr"))
+	}
 	if (props.SFELLOAD)
 		compile.dd(new DDStatement().dsn(props.SFELLOAD).options("shr"))
 

languages/Transfer.groovy

@@ -83,6 +83,17 @@ buildList.each { buildFile ->
 				props.error = "true"
 				buildUtils.updateBuildResult(errorMsg:errorMsg)
 			} 
+
+            // get copy mode value from Property Mappings
+			def copyMode = props.getFileProperty('transfer_copyMode', buildFile)
+
+            DBBConstants.CopyMode transferCopyMode
+            if (copyMode != null) {
+                transferCopyMode = DBBConstants.CopyMode.valueOf(copyMode)
+            } else {
+                transferCopyMode = DBBConstants.CopyMode.valueOf("TEXT")
+                if (props.verbose) println "** No CopyMode found for file '${buildFile}'. Using 'TEXT' as default."
+            }
 
 			// allocate target dataset
 			if (!verifiedBuildDatasets.contains(targetDataset)) { // using a cache not to allocate all defined datasets
@@ -93,9 +104,16 @@ buildList.each { buildFile ->
 			// copy the file to the target dataset
 			String deployType = buildUtils.getDeployType("transfer", buildFile, null)
 
-			try {
-				int rc = new CopyToPDS().key(buildFile).file(new File(buildUtils.getAbsolutePath(buildFile))).dataset(targetDataset).member(member).output(true).deployType(deployType).execute()
-				if (props.verbose) println "** Copied $buildFile to $targetDataset with deployType $deployType (rc = $rc)"
+			try {				
+				int rc = new CopyToPDS().key(buildFile)
+                    .file(new File(buildUtils.getAbsolutePath(buildFile)))
+                    .copyMode(transferCopyMode)
+                    .dataset(targetDataset)
+                    .member(member)
+                    .output(true)
+                    .deployType(deployType)
+                    .execute()
+                if (props.verbose) println "** Copied $buildFile to $targetDataset with copyMode $transferCopyMode and deployType $deployType (rc = $rc)"				
 
 				if (rc!=0){
 					String errorMsg = "*! The CopyToPDS return code ($rc) for $buildFile exceeded the maximum return code allowed (0)."

samples/MortgageApplication/crb/cics-resourcesDef.yaml

@@ -2,6 +2,9 @@ resourceDefinitions:
 - mapset:
     name: EPSMORT
     group: EPSMTM
+- mapset:
+    name: EPSMLIS
+    group: EPSMTM
 - program:
     name: EPSCMORT
     group: EPSMTM

samples/application-conf/MFS.properties

@@ -11,6 +11,11 @@ mfs_fileBuildRank=
 mfs_phase1MaxRC=4
 mfs_phase2MaxRC=4
 
+#
+# skip execution of phase2 translation
+# always execute phase 1 conversion for syntax validation
+# can be overridden by file properties
+mfs_phase2Execution=false
 
 #
 # default MFS parameters
@@ -24,4 +29,4 @@ mfs_impactPropertyList=mfs_phase1Parms,mfs_phase2Parms
 
 #
 # default deployType
-mfs_deployType=MFSMAPLOAD
+mfs_deployType=MFSSOURCE