Fix ArrayList error in Orchestration pipeline - 4.x

CHANGELOG.md

@@ -54,6 +54,7 @@
 - Refactor and create unit test for LeVADocumentUseCase.getReferencedDocumentsVersion ([#744](https://github.com/opendevstack/ods-jenkins-shared-library/pull/744))
 - Remove non breakable white space from Json response of JiraService ([760](https://github.com/opendevstack/ods-jenkins-shared-library/pull/760))
 - Rollback changes filtering components for TIP ([#762](https://github.com/opendevstack/ods-jenkins-shared-library/pull/762))
+- Fix SerializationException, Fix NULL in SSDS generation and Fix SaaS bug in FinalizeStage ([#756](https://github.com/opendevstack/ods-jenkins-shared-library/pull/756))
 
 ## [3.0] - 2020-08-11
 

src/org/ods/component/Context.groovy

@@ -1,5 +1,6 @@
 package org.ods.component
 
+import org.ods.util.ShellWithRetry
 import org.ods.util.Logger
 import org.ods.services.ServiceRegistry
 import org.ods.services.BitbucketService
@@ -9,10 +10,14 @@ import org.ods.services.OpenShiftService
 import com.cloudbees.groovy.cps.NonCPS
 import groovy.json.JsonSlurperClassic
 import groovy.json.JsonOutput
+import java.util.concurrent.ExecutionException
 
 @SuppressWarnings(['MethodCount', 'UnnecessaryObjectReferences'])
 class Context implements IContext {
 
+    static final int MAX_RETRIES = 5
+    static final int WAIT_TIME_SECONDS = 1
+
     final List excludeFromContextDebugConfig = ['nexusPassword', 'nexusUsername']
     // script is the context of the Jenkinsfile. That means that things like "sh" need to be called on script.
     private final def script
@@ -34,8 +39,27 @@ class Context implements IContext {
         this.localCheckoutEnabled = localCheckoutEnabled
     }
 
-    @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'MethodSize', 'Instanceof'])
     def assemble() {
+        int retry = 0
+        boolean executedWithErrors = true
+
+        while (executedWithErrors && retry++ < MAX_RETRIES) {
+            try {
+                assembleWithRetry()
+                executedWithErrors = false
+            } catch (java.io.NotSerializableException err) {
+                logger.warn ("WARN: Jenkins serialization issue; attempt #: ${retry}, when: context.assemble()")
+                script.sleep(WAIT_TIME_SECONDS)
+            }
+        }
+
+        if (executedWithErrors) {
+            throw new ExecutionException("Jenkins serialization issue, when: context.assemble()")
+        }
+    }
+
+    @SuppressWarnings(['AbcMetric', 'CyclomaticComplexity', 'MethodSize', 'Instanceof'])
+    def assembleWithRetry() {
         logger.debug 'Validating input ...'
         // branchToEnvironmentMapping must be given, but it is OK to be empty - e.g.
         // if the repository should not be deployed to OpenShift at all.
@@ -60,8 +84,8 @@ class Context implements IContext {
         config.sonarQubeEdition = script.env.SONAR_EDITION ?: 'community'
 
         config.globalExtensionImageLabels = getExtensionBuildParams()
-        config.globalExtensionImageLabels << getEnvParamsAndAddPrefix('OPENSHIFT_BUILD',
-            'JENKINS_MASTER_')
+        config.globalExtensionImageLabels.putAll(getEnvParamsAndAddPrefix('OPENSHIFT_BUILD',
+            'JENKINS_MASTER_'))
 
         logger.debug("Got external build labels: ${config.globalExtensionImageLabels}")
 
@@ -164,8 +188,8 @@ class Context implements IContext {
             config.globalExtensionImageLabels = [:]
         }
         // get the build labels from the env running in ..
-        config.globalExtensionImageLabels << getEnvParamsAndAddPrefix('OPENSHIFT_BUILD',
-            'JENKINS_AGENT_')
+        config.globalExtensionImageLabels.putAll(getEnvParamsAndAddPrefix('OPENSHIFT_BUILD',
+            'JENKINS_AGENT_'))
     }
 
     boolean getDebug() {
@@ -516,20 +540,27 @@ class Context implements IContext {
     }
 
     Map<String,String> getEnvParamsAndAddPrefix (String envNamePattern = 'ods.build.', String keyPrefix = '') {
-        String rawEnv = script.sh(
-            returnStdout: true, script: "env | grep ${envNamePattern} || true",
-            label: 'getting extension labels from current environment'
-          ).trim()
+        String rawEnv = new ShellWithRetry(script, logger).execute(
+            returnStdout: true,
+            script: "env | grep ${envNamePattern} || true",
+            label: 'getting extension labels from current environment')
 
-        if (rawEnv.size() == 0 ) {
+        if (rawEnv.length() == 0 ) {
             return [:]
         }
 
-        return rawEnv.normalize().split(System.getProperty('line.separator')).inject([ : ] ) { kvMap, line ->
-            Iterator kv = line.toString().tokenize('=').iterator()
-            kvMap.put(keyPrefix + kv.next(), kv.hasNext() ? kv.next() : '')
-            kvMap
+        return normalizeEnvironment(rawEnv, keyPrefix)
+    }
+
+    @NonCPS
+    Map<String,String> normalizeEnvironment (String rawEnv, String keyPrefix) {
+        def lineSplitEnv = rawEnv.normalize().split(System.getProperty('line.separator'))
+        Map normalizedEnv = [ : ]
+        for (int lineC = 0; lineC < lineSplitEnv.size(); lineC++) {
+            def splittedLine = lineSplitEnv[lineC].toString().tokenize('=')
+            normalizedEnv.put(keyPrefix + splittedLine[0], splittedLine[1])
         }
+        return normalizedEnv
     }
 
     String getOpenshiftApplicationDomain () {
@@ -616,7 +647,7 @@ class Context implements IContext {
 
     private String commitHashForBuild(build) {
         return build
-            .getActions(hudson.plugins.git.util.BuildData.class)
+            .getActions(hudson.plugins.git.util.BuildData)
             .find { action -> action.getRemoteUrls().contains(config.gitUrl) }
             .getLastBuiltRevision().getSha1String()
     }

src/org/ods/component/IContext.groovy

@@ -197,4 +197,5 @@ interface IContext {
 
     // get commit the working tree
     boolean getCommitGitWorkingTree ()
+
 }

src/org/ods/component/Pipeline.groovy

@@ -100,7 +100,9 @@ class Pipeline implements Serializable {
                                 replace(wtfEnvBug, "${defaultDockerRegistry}/")
                             logger.warn ("Patched image via master env to: ${config.image}")
                         }
+
                         context.assemble()
+
                         // register services after context was assembled
                         logger.debug('-> Registering & loading global services')
                         def registry = ServiceRegistry.instance
@@ -356,7 +358,7 @@ class Pipeline implements Serializable {
         return this.ciSkipEnabled && gitService.ciSkipInCommitMessage
     }
 
-    private def prepareAgentPodConfig(Map config) {
+    private void prepareAgentPodConfig(Map config) {
         if (!config.image && !config.imageStreamTag && !config.podContainers) {
             script.error "One of 'image', 'imageStreamTag' or 'podContainers' is required"
         }

src/org/ods/orchestration/BuildStage.groovy

@@ -57,12 +57,14 @@ class BuildStage extends Stage {
                         "- no unit tests results will be reported")
                 }
 
+                logger.info("levaDocScheduler.run start")
                 levaDocScheduler.run(
                     phase,
                     MROPipelineUtil.PipelinePhaseLifecycleStage.POST_EXECUTE_REPO,
                     repo,
                     data
                 )
+                logger.info("levaDocScheduler.run end")
             }
         }
 
@@ -88,7 +90,9 @@ class BuildStage extends Stage {
         def failedRepos = repos.flatten().findAll { it.data?.failedStage }
         if (project.isAssembleMode && project.isWorkInProgress &&
             (project.hasFailingTests() || failedRepos.size > 0)) {
-            util.failBuild("Failing build as repositories contain errors!\nFailed: ${failedRepos}")
+            def errMessage = "Failing build as repositories contain errors!\nFailed: ${failedRepos}"
+            util.failBuild(errMessage)
+            throw new IllegalStateException(errMessage)
         }
     }
 

src/org/ods/orchestration/DeployStage.groovy

@@ -151,4 +151,5 @@ class DeployStage extends Stage {
             logger.warn("No log state for ${repo.data} found!")
         }
     }
+
 }

src/org/ods/orchestration/FinalizeStage.groovy

@@ -63,11 +63,12 @@ class FinalizeStage extends Stage {
             }
             repoFinalizeTasks.failFast = true
             script.parallel(repoFinalizeTasks)
-
+            logger.debug("Integrate into main branch")
             if (project.isAssembleMode && !project.isWorkInProgress) {
                 integrateIntoMainBranchRepos(steps, git)
             }
 
+            logger.debug("Gatering commits")
             gatherCreatedExecutionCommits(steps, git)
 
             if (!project.buildParams.rePromote) {
@@ -131,21 +132,23 @@ class FinalizeStage extends Stage {
 
     private void pushRepos(IPipelineSteps steps, GitService git) {
         def flattenedRepos = repos.flatten()
-        def repoPushTasks = flattenedRepos.collectEntries { repo ->
-            [
-                (repo.id): {
-                    steps.dir("${steps.env.WORKSPACE}/${MROPipelineUtil.REPOS_BASE_DIR}/${repo.id}") {
-                        if (project.isWorkInProgress) {
-                            git.pushRef(repo.branch)
-                        } else if (project.isAssembleMode) {
-                            git.createTag(project.targetTag)
-                            git.pushBranchWithTags(project.gitReleaseBranch)
-                        } else {
-                            git.createTag(project.targetTag)
-                            git.pushRef(project.targetTag)
-                        }
+        def repoPushTasks = [ : ]
+        def repoSize = flattenedRepos.size()
+        for (def i = 0; i < repoSize; i++) {
+            def repo = flattenedRepos[i]
+            repoPushTasks << [ (repo.id): {
+                steps.dir("${steps.env.WORKSPACE}/${MROPipelineUtil.REPOS_BASE_DIR}/${repo.id}") {
+                    if (project.isWorkInProgress) {
+                        git.pushRef(repo.branch)
+                    } else if (project.isAssembleMode) {
+                        git.createTag(project.targetTag)
+                        git.pushBranchWithTags(project.gitReleaseBranch)
+                    } else {
+                        git.createTag(project.targetTag)
+                        git.pushRef(project.targetTag)
                     }
                 }
+            }
             ]
         }
         repoPushTasks.failFast = true
@@ -154,42 +157,48 @@ class FinalizeStage extends Stage {
 
     private void gatherCreatedExecutionCommits(IPipelineSteps steps, GitService git) {
         def flattenedRepos = repos.flatten()
-        def gatherCommitTasks = flattenedRepos.collectEntries { repo ->
-            [
-                (repo.id): {
-                    steps.dir("${steps.env.WORKSPACE}/${MROPipelineUtil.REPOS_BASE_DIR}/${repo.id}") {
-                        repo.data.git.createdExecutionCommit = git.commitSha
-                    }
+        def gatherCommitTasks = [ : ]
+        def repoSize = flattenedRepos.size()
+        for (def i = 0; i < repoSize; i++) {
+            def repo = flattenedRepos[i]
+            gatherCommitTasks << [ (repo.id): {
+                steps.dir("${steps.env.WORKSPACE}/${MROPipelineUtil.REPOS_BASE_DIR}/${repo.id}") {
+                    repo.data.git.createdExecutionCommit = git.commitSha
+                    steps.echo "repo.id: ${repo.id}: ${repo.data.git.createdExecutionCommit}"
                 }
+            }
             ]
         }
+
         gatherCommitTasks.failFast = true
         script.parallel(gatherCommitTasks)
     }
 
     private void integrateIntoMainBranchRepos(IPipelineSteps steps, GitService git) {
         def flattenedRepos = repos.flatten()
-        def repoIntegrateTasks = flattenedRepos
-            .findAll { it.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_TEST &&
-               it.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_INFRA &&
-               it.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_SAAS_SERVICE }
-            .collectEntries { repo ->
-                [
-                    (repo.id): {
-                        steps.dir("${steps.env.WORKSPACE}/${MROPipelineUtil.REPOS_BASE_DIR}/${repo.id}") {
-                            def filesToCheckout = []
-                            if (steps.fileExists('openshift')) {
-                                filesToCheckout = ['openshift/ods-deployments.json']
-                            } else {
-                                filesToCheckout = [
-                                    'openshift-exported/ods-deployments.json',
-                                    'openshift-exported/template.yml'
-                                ]
-                            }
-                            git.mergeIntoMainBranch(project.gitReleaseBranch, repo.branch, filesToCheckout)
+        def repoIntegrateTasks = [ : ]
+        def repoSize = flattenedRepos.size()
+        for (def i = 0; i < repoSize; i++) {
+            def repo = flattenedRepos[i]
+            if (repo.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_TEST &&
+                repo.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_INFRA &&
+                repo.type?.toLowerCase() != MROPipelineUtil.PipelineConfig.REPO_TYPE_ODS_SAAS_SERVICE ) {
+                repoIntegrateTasks << [ (repo.id): {
+                    steps.dir("${steps.env.WORKSPACE}/${MROPipelineUtil.REPOS_BASE_DIR}/${repo.id}") {
+                        def filesToCheckout = []
+                        if (steps.fileExists('openshift')) {
+                            filesToCheckout = ['openshift/ods-deployments.json']
+                        } else {
+                            filesToCheckout = [
+                                'openshift-exported/ods-deployments.json',
+                                'openshift-exported/template.yml'
+                            ]
                         }
+                        git.mergeIntoMainBranch(project.gitReleaseBranch, repo.branch, filesToCheckout)
                     }
+                }
                 ]
+            }
         }
         repoIntegrateTasks.failFast = true
         script.parallel(repoIntegrateTasks)
@@ -199,10 +208,15 @@ class FinalizeStage extends Stage {
         // record release manager repo state
         logger.debug "Finalize: Recording HEAD commits from repos ..."
         logger.debug "On release manager commit ${git.commitSha}"
-        def gitHeads = repos.flatten().collectEntries { repo ->
+        def flattenedRepos = repos.flatten()
+        def gitHeads = [ : ]
+        def repoSize = flattenedRepos.size()
+        for (def i = 0; i < repoSize; i++) {
+            def repo = flattenedRepos[i]
             logger.debug "HEAD of repo '${repo.id}': ${repo.data.git.createdExecutionCommit}"
-            [(repo.id): (repo.data.git.createdExecutionCommit ?: '')]
+            gitHeads << [ (repo.id): (repo.data.git.createdExecutionCommit ?: '')]
         }
+
         def envState = [
             version: project.buildParams.version,
             changeId: project.buildParams.changeId,