Merge branch 'master' into stagesetup

bin/server

@@ -229,6 +229,16 @@ const buildClusterFactory = Models.BuildClusterFactory.getInstance({
     datastoreRO,
     scm
 });
+const pipelineTemplateFactory = Models.PipelineTemplateFactory.getInstance({
+    datastore,
+    datastoreRO,
+    scm
+});
+const pipelineTemplateVersionFactory = Models.PipelineTemplateVersionFactory.getInstance({
+    datastore,
+    datastoreRO,
+    scm
+});
 
 // @TODO run setup for SCM and Executor
 // datastoreConfig.ddlSync => sync datastore schema (ddl) via api (default: true)
@@ -244,6 +254,8 @@ datastore.setup(datastoreConfig.ddlSyncEnabled).then(() =>
         commandFactory,
         commandTagFactory,
         templateFactory,
+        pipelineTemplateFactory,
+        pipelineTemplateVersionFactory,
         templateTagFactory,
         pipelineFactory,
         jobFactory,

lib/server.js

@@ -126,6 +126,11 @@ module.exports = async config => {
             commandTagFactory: config.commandTagFactory,
             templateFactory: config.templateFactory,
             templateTagFactory: config.templateTagFactory,
+            pipelineTemplateFactory: config.pipelineTemplateFactory,
+            pipelineTemplateVersionFactory: config.pipelineTemplateVersionFactory,
+            templateMetaFactory: config.templateMetaFactory,
+            jobTemplateTagFactory: config.jobTemplateTagFactory,
+            pipelineTemplateTagFactory: config.pipelineTemplateTagFactory,
             stageFactory: config.stageFactory,
             stageBuildFactory: config.stageBuildFactory,
             triggerFactory: config.triggerFactory,

package.json

@@ -105,7 +105,7 @@
     "screwdriver-config-parser": "^8.0.1",
     "screwdriver-coverage-bookend": "^2.0.0",
     "screwdriver-coverage-sonar": "^4.1.1",
-    "screwdriver-data-schema": "^22.9.8",
+    "screwdriver-data-schema": "^22.9.12",
     "screwdriver-datastore-sequelize": "^8.0.0",
     "screwdriver-executor-base": "^9.0.1",
     "screwdriver-executor-docker": "^6.0.0",
@@ -118,12 +118,12 @@
     "screwdriver-notifications-email": "^3.0.0",
     "screwdriver-notifications-slack": "^5.0.0",
     "screwdriver-request": "^2.0.1",
-    "screwdriver-scm-base": "^8.1.0",
+    "screwdriver-scm-base": "^8.1.1",
     "screwdriver-scm-bitbucket": "^5.0.1",
-    "screwdriver-scm-github": "^12.1.1",
+    "screwdriver-scm-github": "^12.2.4",
     "screwdriver-scm-gitlab": "^3.1.0",
     "screwdriver-scm-router": "^7.0.0",
-    "screwdriver-template-validator": "^6.0.0",
+    "screwdriver-template-validator": "^7.0.0",
     "screwdriver-workflow-parser": "^4.1.1",
     "sqlite3": "^5.1.4",
     "stream": "0.0.2",

plugins/builds/index.js

@@ -1068,7 +1068,7 @@ const buildsPlugin = {
                  */
                 const isORTrigger = !joinListNames.includes(current.job.name);
 
-                if (joinListNames.length === 0 || isORTrigger) {
+                if (isORTrigger) {
                     const internalBuildConfig = {
                         jobFactory,
                         buildFactory,
@@ -1177,12 +1177,27 @@ const buildsPlugin = {
                     finishedInternalBuilds = finishedInternalBuilds.concat(parallelBuilds);
                 }
 
-                fillParentBuilds(parentBuilds, current, finishedInternalBuilds);
-                // If next build is internal, look at the finished builds for this event
                 const nextJobId = joinObj[nextJobName].id;
-                const nextBuild = finishedInternalBuilds.find(
-                    b => b.jobId === nextJobId && b.eventId === current.event.id
-                );
+
+                let nextBuild;
+
+                // If next build is internal, look at the finished builds for this event
+                nextBuild = finishedInternalBuilds.find(b => b.jobId === nextJobId && b.eventId === current.event.id);
+
+                if (!nextBuild) {
+                    // If the build to join fails and it succeeds on restart, depending on the timing, the latest build will be that of a child event.
+                    // In that case, `nextBuild` will be null and will not be triggered even though there is a build that should be triggered.
+                    // Now we need to check for the existence of a build that should be triggered in its own event.
+                    nextBuild = await buildFactory.get({
+                        eventId: current.event.id,
+                        jobId: nextJobId
+                    });
+
+                    finishedInternalBuilds = finishedInternalBuilds.concat(nextBuild);
+                }
+
+                fillParentBuilds(parentBuilds, current, finishedInternalBuilds);
+
                 let newBuild;
 
                 // Create next build
@@ -1328,6 +1343,7 @@ const buildsPlugin = {
 
                         const joinList = nextJobs[nextJobName].join;
                         const joinListNames = joinList.map(j => j.name);
+                        const isORTrigger = !joinListNames.includes(triggerName);
 
                         if (nextBuild) {
                             // update current build info in parentBuilds
@@ -1363,6 +1379,17 @@ const buildsPlugin = {
                             });
                         }
 
+                        if (isORTrigger) {
+                            if (!['CREATED', null, undefined].includes(newBuild.status)) {
+                                return newBuild;
+                            }
+
+                            newBuild.status = 'QUEUED';
+                            await newBuild.update();
+
+                            return newBuild.start();
+                        }
+
                         const { hasFailure, done } = await getParentBuildStatus({
                             newBuild,
                             joinListNames,

plugins/pipelines/README.md

@@ -206,3 +206,117 @@ handler: async (request, h) => {
     // ...
 }
 ```
+
+#### Pipeline Templates
+##### Get all pipeline templates
+
+`GET /pipeline/templates`
+
+Can use additional options for sorting, pagination and count:
+`GET /pipeline/templates?sort=ascending&sortBy=name&page=1&count=50`
+
+##### Get all versions for a pipeline template
+
+`GET /pipeline/templates/{namespace}/{name}/versions`
+
+Can use additional options for sorting, pagination and count:
+`GET /pipeline/templates/{namespace}/{name}/versions?sort=ascending&page=1&count=50`
+
+##### Create a pipeline template
+Creating a template will store the template meta (`name`, `namespace`, `maintainer`, `latestVersion`, `trustedSinceVersion`, `pipelineId`) and template version (`description`, `version`, `config`, `createTime`, `templateId`)  into the datastore.
+
+`version` will be auto-bumped. For example, if `mypipelinetemplate@1.0.0` already exists and the version passed in is `1.0.0`, the newly created template will be version `1.0.1`.
+
+
+`POST /pipeline/template`
+###### Arguments
+
+'name', 'namespace', 'version', 'description', 'maintainer', 'config'
+
+* `name` - Name of the template
+* `namespace` - Namespace of the template
+* `version` - Version of the template
+* `description` - Description of the template
+* `maintainer` - Maintainer of the template
+* `config` - Config of the template. This field is an object that includes `steps`, `image`, and optional `secrets`, `environments`. Similar to what's inside the `pipeline`
+
+Example payload:
+```json
+{
+    "name": "example-template",
+    "namespace": "my-namespace",
+    "version": "1.3.1",
+    "description": "An example template",
+    "maintainer": "example@gmail.com",
+    "config": {
+        "steps": [{
+            "echo": "echo hello"
+        }]
+    }
+}
+```
+
+##### Validate a pipeline template
+Validate a pipeline template and return a JSON containing the boolean property ‘valid’ indicating if the template is valid or not
+
+`POST /pipeline/template/validate`
+
+###### Arguments
+
+'name', 'namespace', 'version', 'description', 'maintainer', 'config'
+
+* `name` - Name of the template
+* `namespace` - Namespace of the template
+* `version` - Version of the template
+* `description` - Description of the template
+* `maintainer` - Maintainer of the template
+* `config` - Config of the template. This field is an object that includes `steps`, `image`, and optional `secrets`, `environments`. Similar to what's inside the `pipeline`
+
+Example payload:
+```json
+{
+    "name": "example-template",
+    "namespace": "my-namespace",
+    "version": "1.3.1",
+    "description": "An example template",
+    "maintainer": "example@gmail.com",
+    "config": {
+        "steps": [{
+            "echo": "echo hello"
+        }]
+    }
+}
+```
+
+#### Get a pipeline template by namespace and name
+
+`GET /pipeline/template/{namespace}/{name}`
+
+##### Get a specific pipeline template by id
+
+`GET /pipeline/template/{id}`
+
+##### Get version of a pipeline template by name, namespace, version or tag
+
+`GET /pipeline/template/{namespace}/{name}/{versionOrTag}`
+
+
+#### Template Tag
+Template tag allows fetching on template version by tag. For example, tag `mytemplate@1.1.0` as `stable`.
+
+##### Get all tags for a pipeline template by name, namespace
+
+`GET /pipeline/templates/{namespace}/{name}/tags`
+
+Can use additional options for sorting, pagination and count:
+`GET /pipeline/templates/{namespace}/{name}/tags?sort=ascending&sortBy=name&page=1&count=50`
+
+##### Create/Update a tag
+
+If the template tag already exists, it will update the tag with the new version. If the template tag doesn't exist yet, this endpoint will create the tag.
+
+*Note: This endpoint is only accessible in `build` scope and the permission is tied to the pipeline that creates the template.*
+
+`PUT /templates/{templateName}/tags/{tagName}` with the following payload
+
+* `version` - Exact version of the template (ex: `1.1.0`)

plugins/pipelines/index.js

@@ -29,6 +29,15 @@ const latestCommitEvent = require('./latestCommitEvent');
 const getAdmin = require('./admins/get');
 const deleteCache = require('./caches/delete');
 const openPrRoute = require('./openPr');
+const createTemplateRoute = require('./templates/create');
+const validateTemplateRoute = require('./templates/validate');
+const listTemplatesRoute = require('./templates/list');
+const listTemplateVersionsRoute = require('./templates/listVersions');
+const listTagsRoute = require('./templates/listTags');
+const getTemplateRoute = require('./templates/get');
+const getTemplateByIdRoute = require('./templates/getTemplateById');
+const createTagRoute = require('./templates/createTag');
+const getVersionRoute = require('./templates/getVersion');
 
 /**
  * Pipeline API Plugin
@@ -195,7 +204,16 @@ const pipelinesPlugin = {
             latestCommitEvent(),
             getAdmin(),
             deleteCache(),
-            openPrRoute()
+            openPrRoute(),
+            createTemplateRoute(),
+            validateTemplateRoute(),
+            listTemplatesRoute(),
+            listTemplateVersionsRoute(),
+            listTagsRoute(),
+            getVersionRoute(),
+            getTemplateByIdRoute(),
+            getTemplateRoute(),
+            createTagRoute()
         ]);
     }
 };

plugins/pipelines/templates/create.js

@@ -0,0 +1,61 @@
+'use strict';
+
+const boom = require('@hapi/boom');
+const schema = require('screwdriver-data-schema');
+const validator = require('screwdriver-template-validator').parsePipelineTemplate;
+const templateSchema = schema.api.templateValidator;
+
+module.exports = () => ({
+    method: 'POST',
+    path: '/pipeline/template',
+    options: {
+        description: 'Create a new pipeline template',
+        notes: 'Create a specific pipeline template',
+        tags: ['api', 'pipelineTemplate'],
+        auth: {
+            strategies: ['token'],
+            scope: ['build']
+        },
+
+        handler: async (request, h) => {
+            const { pipelineTemplateVersionFactory, pipelineTemplateFactory } = request.server.app;
+
+            const config = await validator(request.payload.yaml);
+
+            if (config.errors.length > 0) {
+                throw boom.badRequest(`Template has invalid format: ${config.errors.length} error(s).`, config.errors);
+            }
+
+            const pipelineTemplate = await pipelineTemplateFactory.get({
+                name: config.template.name,
+                namespace: config.template.namespace
+            });
+
+            const { isPR, pipelineId } = request.auth.credentials;
+
+            // If template name exists, but this build's pipelineId is not the same as template's pipelineId
+            // Then this build does not have permission to publish
+            if (isPR || (pipelineTemplate && pipelineId !== pipelineTemplate.pipelineId)) {
+                throw boom.forbidden('Not allowed to publish this template');
+            }
+
+            const template = await pipelineTemplateVersionFactory.create(
+                {
+                    ...config.template,
+                    pipelineId
+                },
+                pipelineTemplateFactory
+            );
+
+            const location = new URL(
+                `${request.path}/${template.id}`,
+                `${request.server.info.protocol}://${request.headers.host}`
+            ).toString();
+
+            return h.response(template.toJson()).header('Location', location).code(201);
+        },
+        validate: {
+            payload: templateSchema.input
+        }
+    }
+});