feat(2470): Skip execution of virtual jobs

plugins/builds/index.js

@@ -34,7 +34,8 @@ const {
     createExternalEvent,
     getBuildsForGroupEvent,
     buildsToRestartFilter,
-    trimJobName
+    trimJobName,
+    Status
 } = require('./triggers/helpers');
 
 /**
@@ -86,10 +87,14 @@ async function triggerNextJobs(config, app) {
     const andTrigger = new AndTrigger(app, config, currentEvent);
     const currentPipelineNextJobs = extractCurrentPipelineJoinData(pipelineJoinData, currentPipeline.id);
 
+    const downstreamOfNextJobsToBeProcessed = [];
+
     for (const [nextJobName, nextJob] of Object.entries(currentPipelineNextJobs)) {
         const nextJobId = nextJob.id || (await getJobId(nextJobName, currentPipeline.id, jobFactory));
+        const { isVirtual: isNextJobVirtual, stageName: nextJobStageName } = nextJob;
         const resource = `pipeline:${currentPipeline.id}:event:${currentEvent.id}`;
         let lock;
+        let nextBuild;
 
         try {
             lock = await locker.lock(resource);
@@ -109,9 +114,34 @@ async function triggerNextJobs(config, app) {
              *    joinList doesn't include D, so start A
              */
             if (isOrTrigger(currentEvent.workflowGraph, originalCurrentJobName, trimJobName(nextJobName))) {
-                await orTrigger.execute(currentEvent, currentPipeline.id, nextJobName, nextJobId, parentBuilds);
+                nextBuild = await orTrigger.execute(
+                    currentEvent,
+                    currentPipeline.id,
+                    nextJobName,
+                    nextJobId,
+                    parentBuilds,
+                    isNextJobVirtual
+                );
             } else {
-                await andTrigger.execute(nextJobName, nextJobId, parentBuilds, joinListNames);
+                nextBuild = await andTrigger.execute(
+                    nextJobName,
+                    nextJobId,
+                    parentBuilds,
+                    joinListNames,
+                    isNextJobVirtual,
+                    nextJobStageName
+                );
+            }
+
+            if (isNextJobVirtual && nextBuild.status === Status.SUCCESS) {
+                downstreamOfNextJobsToBeProcessed.push({
+                    build: nextBuild,
+                    event: currentEvent,
+                    job: await nextBuild.job,
+                    pipeline: currentPipeline,
+                    scmContext: config.scmContext,
+                    username: config.username
+                });
             }
         } catch (err) {
             logger.error(
@@ -186,6 +216,7 @@ async function triggerNextJobs(config, app) {
 
         for (const [nextJobName, nextJob] of Object.entries(joinedPipeline.jobs)) {
             const nextJobId = nextJob.id || (await getJobId(nextJobName, currentPipeline.id, jobFactory));
+            const { isVirtual: isNextJobVirtual, stageName: nextJobStageName } = nextJob;
 
             const { parentBuilds } = parseJobInfo({
                 joinObj: joinedPipeline.jobs,
@@ -196,16 +227,19 @@ async function triggerNextJobs(config, app) {
                 nextPipelineId: joinedPipelineId
             });
 
+            let nextBuild;
+
             try {
                 if (resource) lock = await locker.lock(resource);
 
                 if (isOrTrigger(externalEvent.workflowGraph, remoteTriggerName, nextJobName)) {
-                    await remoteTrigger.execute(
+                    nextBuild = await remoteTrigger.execute(
                         externalEvent,
                         externalEvent.pipelineId,
                         nextJobName,
                         nextJobId,
-                        parentBuilds
+                        parentBuilds,
+                        isNextJobVirtual
                     );
                 } else {
                     // Re get join list when first time remote trigger since external event was empty and cannot get workflow graph then
@@ -215,15 +249,30 @@ async function triggerNextJobs(config, app) {
                             : workflowParser.getSrcForJoin(externalEvent.workflowGraph, { jobName: nextJobName });
                     const joinListNames = joinList.map(j => j.name);
 
-                    await remoteJoin.execute(
+                    nextBuild = await remoteJoin.execute(
                         externalEvent,
                         nextJobName,
                         nextJobId,
                         parentBuilds,
                         groupEventBuilds,
-                        joinListNames
+                        joinListNames,
+                        isNextJobVirtual,
+                        nextJobStageName
                     );
                 }
+
+                if (isNextJobVirtual && nextBuild.status === Status.SUCCESS) {
+                    const nextJobModel = await nextBuild.job;
+
+                    downstreamOfNextJobsToBeProcessed.push({
+                        build: nextBuild,
+                        event: currentEvent,
+                        job: nextJobModel,
+                        pipeline: await nextJobModel.pipeline,
+                        scmContext: config.scmContext,
+                        username: config.username
+                    });
+                }
             } catch (err) {
                 logger.error(
                     `Error in triggerJobsInExternalPipeline:${joinedPipelineId} from pipeline:${currentPipeline.id}-${currentJob.name}-event:${currentEvent.id} `,
@@ -235,6 +284,10 @@ async function triggerNextJobs(config, app) {
         }
     }
 
+    for (const nextConfig of downstreamOfNextJobsToBeProcessed) {
+        await triggerNextJobs(nextConfig, app);
+    }
+
     return null;
 }
 

plugins/builds/triggers/and.js

@@ -56,9 +56,11 @@ class AndTrigger extends JoinBase {
      * @param {String} nextJobId
      * @param {Record<String, Object>} parentBuilds
      * @param {String[]} joinListNames
+     * @param {Boolean} isNextJobVirtual
+     * @param {String} nextJobStageName
      * @returns {Promise<Build>}
      */
-    async execute(nextJobName, nextJobId, parentBuilds, joinListNames) {
+    async execute(nextJobName, nextJobId, parentBuilds, joinListNames, isNextJobVirtual, nextJobStageName) {
         logger.info(`Fetching finished builds for event ${this.currentEvent.id}`);
 
         const relatedBuilds = await this.fetchRelatedBuilds();
@@ -90,7 +92,9 @@ class AndTrigger extends JoinBase {
             nextJobId,
             parentBuilds: newParentBuilds,
             parentBuildId: this.currentBuild.id,
-            joinListNames
+            joinListNames,
+            isNextJobVirtual,
+            nextJobStageName
         });
     }
 }

plugins/builds/triggers/helpers.js

@@ -592,17 +592,18 @@ async function getParentBuildStatus({ newBuild, joinListNames, pipelineId, build
  * @param {Build} arg.newBuild Next build
  * @param {String|undefined} arg.jobName Job name
  * @param {String|undefined} arg.pipelineId Pipeline ID
- * @param {Object|undefined} arg.stage Stage
+ * @param {String|undefined} arg.stageName Stage name
+ * @param {Boolean} arg.isVirtualJob If the job is virtual or not
  * @returns {Promise<Build|null>} The newly updated/created build
  */
-async function handleNewBuild({ done, hasFailure, newBuild, jobName, pipelineId, stage }) {
+async function handleNewBuild({ done, hasFailure, newBuild, jobName, pipelineId, stageName, isVirtualJob }) {
     if (!done || Status.isStarted(newBuild.status)) {
         return null;
     }
 
     // Delete new build since previous build failed
     if (hasFailure) {
-        const stageTeardownName = stage ? getFullStageJobName({ stageName: stage.name, jobName: 'teardown' }) : '';
+        const stageTeardownName = stageName ? getFullStageJobName({ stageName, jobName: 'teardown' }) : '';
 
         // New build is not stage teardown job
         if (jobName !== stageTeardownName) {
@@ -615,7 +616,14 @@ async function handleNewBuild({ done, hasFailure, newBuild, jobName, pipelineId,
         return null;
     }
 
-    // All join builds finished successfully and it's clear that a new build has not been started before.
+    // Bypass execution of the build if the job is virtual
+    if (isVirtualJob) {
+        newBuild.status = Status.SUCCESS;
+
+        return newBuild.update();
+    }
+
+    // All join builds finished successfully, and it's clear that a new build has not been started before.
     // Start new build.
     newBuild.status = Status.QUEUED;
     await newBuild.update();
@@ -781,7 +789,11 @@ async function createJoinObject(nextJobNames, current, eventFactory) {
             isExternal = true;
         }
 
-        const jId = event.workflowGraph.nodes.find(n => n.name === trimJobName(jobName)).id;
+        const {
+            id: jId,
+            virtual: isVirtual,
+            stageName
+        } = event.workflowGraph.nodes.find(n => n.name === trimJobName(jobName));
 
         if (!joinObj[nextJobPipelineId]) joinObj[nextJobPipelineId] = {};
         const pipelineObj = joinObj[nextJobPipelineId];
@@ -801,7 +813,7 @@ async function createJoinObject(nextJobNames, current, eventFactory) {
         }
 
         if (!pipelineObj.jobs) pipelineObj.jobs = {};
-        pipelineObj.jobs[nextJobName] = { id: jId, join: jobs, isExternal };
+        pipelineObj.jobs[nextJobName] = { id: jId, join: jobs, isExternal, isVirtual, stageName };
     }
 
     return joinObj;

plugins/builds/triggers/joinBase.js

@@ -9,7 +9,6 @@ const { createInternalBuild, updateParentBuilds, getParentBuildStatus, handleNew
  * @typedef {import('screwdriver-models').JobFactory} JobFactory
  * @typedef {import('screwdriver-models/lib/event')} Event
  * @typedef {import('screwdriver-models/lib/build')} Build
- * @typedef {import('screwdriver-models/lib/stage')} Stage
  */
 
 class JoinBase {
@@ -21,7 +20,6 @@ class JoinBase {
      * @param {JobFactory} app.jobFactory Server app object
      * @param {Object} config Configuration object
      * @param {Build} config.build
-     * @param {Stage} config.stage
      * @param {String} config.username
      * @param {String} config.scmContext
      */
@@ -31,7 +29,6 @@ class JoinBase {
         this.jobFactory = app.jobFactory;
 
         this.currentBuild = config.build;
-        this.stage = config.stage;
         this.username = config.username;
         this.scmContext = config.scmContext;
     }
@@ -47,6 +44,8 @@ class JoinBase {
      * @param {import('./helpers').ParentBuilds} parentBuilds
      * @param {String} parentBuildId
      * @param {String[]} joinListNames
+     * @param {Boolean} isNextJobVirtual
+     * @param {String} nextJobStageName
      * @returns {Promise<Build[]|null>}
      */
     async processNextBuild({
@@ -57,7 +56,9 @@ class JoinBase {
         nextJobId,
         parentBuilds,
         parentBuildId,
-        joinListNames
+        joinListNames,
+        isNextJobVirtual,
+        nextJobStageName
     }) {
         let newBuild;
 
@@ -103,9 +104,10 @@ class JoinBase {
             done,
             hasFailure,
             newBuild,
-            nextJobName,
+            jobName: nextJobName,
             pipelineId,
-            stage: this.stage
+            isVirtualJob: isNextJobVirtual,
+            stageName: nextJobStageName
         });
     }
 }

plugins/builds/triggers/or.js

@@ -15,10 +15,11 @@ class OrTrigger extends OrBase {
      * @param {String} nextJobName
      * @param {Number} nextJobId
      * @param {import('./helpers').ParentBuilds} parentBuilds
+     * @param {Boolean} isNextJobVirtual
      * @return {Promise<Build|null>}
      */
-    async execute(event, pipelineId, nextJobName, nextJobId, parentBuilds) {
-        return this.trigger(event, pipelineId, nextJobName, nextJobId, parentBuilds);
+    async execute(event, pipelineId, nextJobName, nextJobId, parentBuilds, isNextJobVirtual) {
+        return this.trigger(event, pipelineId, nextJobName, nextJobId, parentBuilds, isNextJobVirtual);
     }
 }
 

plugins/builds/triggers/orBase.js

@@ -39,10 +39,11 @@ class OrBase {
      * @param {String} nextJobName
      * @param {Number} nextJobId
      * @param {import('./helpers').ParentBuilds} parentBuilds
+     * @param {Boolean} isNextJobVirtual
      * @return {Promise<Build|null>}
      */
-    async trigger(event, pipelineId, nextJobName, nextJobId, parentBuilds) {
-        const nextBuild = await this.buildFactory.get({
+    async trigger(event, pipelineId, nextJobName, nextJobId, parentBuilds, isNextJobVirtual) {
+        let nextBuild = await this.buildFactory.get({
             eventId: event.id,
             jobId: nextJobId
         });
@@ -52,13 +53,20 @@ class OrBase {
                 return nextBuild;
             }
 
+            // Bypass execution of the build if the job is virtual
+            if (isNextJobVirtual) {
+                nextBuild.status = Status.SUCCESS;
+
+                return nextBuild.update();
+            }
+
             nextBuild.status = Status.QUEUED;
             await nextBuild.update();
 
             return nextBuild.start();
         }
 
-        return createInternalBuild({
+        nextBuild = await createInternalBuild({
             jobFactory: this.jobFactory,
             buildFactory: this.buildFactory,
             pipelineId,
@@ -70,8 +78,17 @@ class OrBase {
             baseBranch: event.baseBranch || null,
             parentBuilds,
             parentBuildId: this.currentBuild.id,
-            start: true
+            start: !isNextJobVirtual
         });
+
+        // Bypass execution of the build if the job is virtual
+        if (isNextJobVirtual) {
+            nextBuild.status = Status.SUCCESS;
+
+            nextBuild = nextBuild.update();
+        }
+
+        return nextBuild;
     }
 }
 

plugins/builds/triggers/remoteJoin.js

@@ -28,9 +28,20 @@ class RemoteJoin extends JoinBase {
      * @param {import('./helpers').ParentBuilds} parentBuilds
      * @param {Build[]} groupEventBuilds Builds of the downstream pipeline, where only the latest ones for each job are included that have the same groupEventId as the externalEvent
      * @param {String[]} joinListNames
+     * @param {Boolean} isNextJobVirtual
+     * @param {String} nextJobStageName
      * @returns {Promise<Build|null>}
      */
-    async execute(externalEvent, nextJobName, nextJobId, parentBuilds, groupEventBuilds, joinListNames) {
+    async execute(
+        externalEvent,
+        nextJobName,
+        nextJobId,
+        parentBuilds,
+        groupEventBuilds,
+        joinListNames,
+        isNextJobVirtual,
+        nextJobStageName
+    ) {
         // fetch builds created due to trigger
         const parallelBuilds = await getParallelBuilds({
             eventFactory: this.eventFactory,
@@ -60,7 +71,9 @@ class RemoteJoin extends JoinBase {
             nextJobId,
             parentBuilds: newParentBuilds,
             parentBuildId,
-            joinListNames
+            joinListNames,
+            isNextJobVirtual,
+            nextJobStageName
         });
     }
 }

plugins/builds/triggers/remoteTrigger.js

@@ -15,10 +15,11 @@ class RemoteTrigger extends OrBase {
      * @param {String} nextJobName
      * @param {String} nextJobId
      * @param {import('./helpers').ParentBuilds} parentBuilds
+     * @param {Boolean} isNextJobVirtual
      * @returns {Promise<Build|null>}
      */
-    async execute(event, pipelineId, nextJobName, nextJobId, parentBuilds) {
-        return this.trigger(event, pipelineId, nextJobName, nextJobId, parentBuilds);
+    async execute(event, pipelineId, nextJobName, nextJobId, parentBuilds, isNextJobVirtual) {
+        return this.trigger(event, pipelineId, nextJobName, nextJobId, parentBuilds, isNextJobVirtual);
     }
 }