diff --git a/src/make-gecko-action-hooks.js b/src/make-gecko-action-hooks.js new file mode 100644 index 0000000..9301459 --- /dev/null +++ b/src/make-gecko-action-hooks.js @@ -0,0 +1,173 @@ +const _ = require('lodash'); +const {getProjects, hgmoPath, scmLevel} = require('./util/projects'); +const {getTaskclusterYml} = require('./util/tcyml'); +const editRole = require('./util/edit-role'); +const editHook = require('./util/edit-hook'); +const {ACTION_HOOKS} = require('./util/action-hooks'); +const chalk = require('chalk'); + +module.exports.setup = (program) => { + return program + .command('make-gecko-action-hooks') + .option('-n, --noop', 'Don\'t change roles, just show difference') + .description('create or update gecko in-tree action hooks'); +}; + +module.exports.run = async function(options) { + let taskcluster = require('taskcluster-client'); + let chalk = require('chalk'); + let projects = await getProjects(); + + // We build action hooks' task definitions from the latest in-tree `.taskcluster.yml`. + const taskclusterYml = { + 'mozilla-central': await getTaskclusterYml(projects['mozilla-central'].repo), + 'comm-central': await getTaskclusterYml(projects['comm-central'].repo), + }; + + for (let action of ACTION_HOOKS) { + const hookGroupId = `project-${action.trustDomain}`; + const hookId = `in-tree-action-${action.level}-${action.actionPerm}`; + const projName = action.trustDomain === 'gecko' ? 'mozilla-central' : 'comm-central'; + const {task, triggerSchema} = makeHookDetails(taskclusterYml[projName], action); + await editHook({ + noop: options.noop, + hookGroupId, + hookId, + metadata: { + name: `Action task ${action.level}-${action.actionPerm}`, + description: [ + '*DO NOT EDIT*', + '', + 'This hook is configured automatically by', + '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', + ].join('\n'), + owner: 'taskcluster-notifications@mozilla.com', + emailOnError: false, // true, TODO + }, + schedule: [], + task, + triggerSchema, + }); + + // make the role with scopes assume:repo::action: for each repo at this level + const projectsAtLevel = Object.keys(projects) + .filter(p => projects[p].access === `scm_level_${action.level}`) + .map(p => projects[p]); + const scopes = projectsAtLevel.map( + project => `assume:repo:hg.mozilla.org/${hgmoPath(project)}:action:${action.actionPerm}`); + await editRole({ + roleId: `hook-id:${hookGroupId}/${hookId}`, + description: [ + '*DO NOT EDIT*', + '', + 'This role is configured automatically by', + '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', + ].join('\n'), + scopes, + noop: options.noop, + }); + } +}; + +const makeHookDetails = (taskclusterYml, action) => { + const task = { + $let: { + tasks_for: 'action', + action: { + name: '${payload.decision.action.name}', + title: '${payload.decision.action.title}', + description: '${payload.decision.action.description}', + taskGroupId: '${payload.decision.action.taskGroupId}', + // Calculate the repo_scope. This is based on user input (the + // repository), but the hooks service checks that this is satisfied by + // the `hook-id:/` role, which is set up above to + // only contain scopes for repositories at this level. Note that the + // actionPerm is *not* based on user input, but is fixed in the + // hookPayload template + repo_scope: 'assume:repo:${payload.decision.repository.url[8:]}:action:' + action.actionPerm, + // cb_name is user-specified for generic actions, but not for those with their own actionPerm + cb_name: action.actionPerm === 'generic' ? + '${payload.decision.action.cb_name}' : + action.actionPerm, + symbol: '${payload.decision.action.symbol}', + }, + push: {$eval: 'payload.decision.push'}, + repository: {$eval: 'payload.decision.repository'}, + input: {$eval: 'payload.user.input'}, + parameters: {$eval: 'payload.decision.parameters'}, + task: {$eval: 'payload.user.task'}, + taskId: {$eval: 'payload.user.taskId'}, + taskGroupId: {$eval: 'payload.user.taskGroupId'}, + // the hooks service provides the value of the new taskId + ownTaskId: {$eval: 'taskId'}, + }, + in: taskclusterYml.tasks[0], + }; + + const objSchema = (attrs, properties) => Object.assign({ + type: 'object', + properties, + additionalProperties: false, + required: Object.keys(properties), + }, attrs); + + const triggerSchema = objSchema({ + description: [ + 'Information required to trigger this hook. This is provided by the `hookPayload`', + 'template in the `actions.json` file generated in-tree.', + ].join(' '), + }, { + decision: objSchema({ + description: [ + 'Information provided by the decision task; this is usually baked into', + '`actions.json`, although any value could be supplied in a direct call to', + '`hooks.triggerHook`.', + ].join(' '), + }, { + action: objSchema({description: 'Information about the action to perform'}, { + name: {type: 'string', description: 'hook name'}, + title: {type: 'string', description: 'hook title'}, + description: {type: 'string', description: 'hook description'}, + taskGroupId: {type: 'string', description: 'taskGroupId of the decision task'}, + cb_name: {type: 'string', description: 'name of the in-tree callback function'}, + symbol: {type: 'string', description: 'treeherder symbol'}, + }), + push: objSchema({description: 'Information about the push that created the decision task'}, { + owner: {type: 'string', description: 'user who made the original push'}, + revision: {type: 'string', description: 'revision of the original push'}, + pushlog_id: {type: 'string', description: 'Mercurial pushlog ID of the original push'}, + }), + repository: objSchema({description: 'Information about the repository where the push occurred'}, { + url: {type: 'string', description: 'repository URL (without trailing slash)', pattern: '[^/]$'}, + project: {type: 'string', description: 'repository project name (also known as "alias")'}, + level: {type: 'string', description: 'repository SCM level'}, + }), + parameters: { + type: 'object', + description: 'decision task parameters', + additionalProperties: true, + }, + }), + user: objSchema({ + description: 'Information provided by the user or user interface', + }, { + input: action.inputSchema ? + action.inputSchema : + { + anyOf: [ + {type: 'object', description: 'user input for the task'}, + {const: null, description: 'null when the action takes no input'}, + ], + }, + taskId: { + anyOf: [ + {type: 'string', description: 'taskId of the task on which this action was activated'}, + {const: null, description: 'null when the action is activated for a taskGroup'}, + ], + }, + taskGroupId: {type: 'string', description: 'taskGroupId on which this action was activated'}, + }), + }); + + return {task, triggerSchema}; +}; diff --git a/src/make-gecko-branch-role.js b/src/make-gecko-branch-role.js index 9185141..67c4989 100644 --- a/src/make-gecko-branch-role.js +++ b/src/make-gecko-branch-role.js @@ -51,8 +51,11 @@ module.exports.run = async function(projectsOption, options) { process.exit(1); } - var roleId = `repo:hg.mozilla.org/${path}:*`; - var scopes = [ + var roleId, scopes, description; + + // repo:* role + roleId = `repo:hg.mozilla.org/${path}:*`; + scopes = [ `assume:${roleRoot}:branch:${domain}:level-${level}:${projectName}`, ]; @@ -66,7 +69,27 @@ module.exports.run = async function(projectsOption, options) { scopes.push(scope); } - var description = [ + description = [ + '*DO NOT EDIT*', + '', + `Scopes for all tasks (cron, action, push) related to https://hg.mozilla.org/${path}`, + '', + 'This role is configured automatically by [taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', + ].join('\n'); + + await editRole({ + roleId, + description, + scopes, + noop: options.noop, + }); + + // repo:branch:default role + roleId = `repo:hg.mozilla.org/${path}:branch:default`; + scopes = [ + `assume:${roleRoot}:push:${domain}:level-${level}:${projectName}`, + ]; + description = [ '*DO NOT EDIT*', '', `Scopes for tasks triggered from pushes to https://hg.mozilla.org/${path}`, @@ -92,7 +115,7 @@ module.exports.run = async function(projectsOption, options) { description = [ '*DO NOT EDIT*', '', - `Scopes for nighlty cron tasks triggered from pushes to https://hg.mozilla.org/${path}`, + `Scopes for nightly cron tasks triggered from pushes to https://hg.mozilla.org/${path}`, '', 'This role is configured automatically by ', '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', diff --git a/src/make-gecko-cron-hook.js b/src/make-gecko-cron-hook.js index 1dd8061..45c1877 100644 --- a/src/make-gecko-cron-hook.js +++ b/src/make-gecko-cron-hook.js @@ -1,8 +1,7 @@ const {getProjects, hgmoPath, scmLevel} = require('./util/projects'); const editRole = require('./util/edit-role'); +const editHook = require('./util/edit-hook'); const chalk = require('chalk'); -const taskcluster = require('taskcluster-client'); -const {diffLines} = require('diff'); module.exports.setup = (program) => { return program @@ -76,9 +75,6 @@ var makeHook = async function(projectName, project, options) { noop: options.noop, }); - // set up hook - - var repo_env, checkout; if (!project.gecko_repo) { // If there isn't a gecko_repo associated with this project, then it is itself a gecko repo @@ -189,54 +185,10 @@ var makeHook = async function(projectName, project, options) { // set a property that is not a valid identifier newHook.task.payload.cache[`level-${level}-checkouts`] = '/builds/worker/checkouts'; - const hooks = new taskcluster.Hooks(); - - let hook; - try { - hook = await hooks.hook(hookGroupId, hookId); - delete hook['hookId']; - delete hook['hookGroupId']; - } catch (err) { - if (err.statusCode !== 404) { - throw err; - } - hook = {}; - } - - // compare and display the differences - const diffs = diffLines( - JSON.stringify(hook, null, 2), - JSON.stringify(newHook, null, 2), - {newlineIsToken: true}); - let diffsFound = false; - diffs.forEach(diff => { - if (diff.added || diff.removed) { - diffsFound = true; - } + await editHook({ + noop: options.noop, + hookGroupId, + hookId, + ...newHook, }); - - if (diffsFound) { - console.log(chalk.green.bold(`changes required for hook ${hookGroupId}/${hookId}:`)); - diffs.forEach(diff => { - if (diff.added) { - diff.value.split(/\n/).forEach(l => console.log(chalk.green('+' + l))); - } else if (diff.removed) { - diff.value.split(/\n/).forEach(l => console.log(chalk.red('-' + l))); - } else { - diff.value.split(/\n/).forEach(l => console.log(' ' + l)); - } - }); - } else { - console.log(chalk.green.bold(`no changes required for hook ${hookGroupId}/${hookId}`)); - } - - if (!options.noop && diffsFound) { - if (hook.task) { - console.log(chalk.green.bold('updating hook')); - await hooks.updateHook(hookGroupId, hookId, newHook); - } else { - console.log(chalk.green.bold('creating hook')); - await hooks.createHook(hookGroupId, hookId, newHook); - } - } }; diff --git a/src/make-gecko-parameterized-roles.js b/src/make-gecko-parameterized-roles.js index bc1bfcb..7e2b309 100644 --- a/src/make-gecko-parameterized-roles.js +++ b/src/make-gecko-parameterized-roles.js @@ -33,7 +33,7 @@ module.exports.run = async function(options) { await editRole({ roleId: `${roleRoot}:branch:${domain}:level-${level}:*`, description: description( - `Scopes for ${domain} projects at level ${level}; the '*' matches the project name.` + `Scopes for tasks associated with all ${domain} projects at level ${level}; the '*' matches the project name.` ), scopes: [ `assume:moz-tree:level:${level}:${domain}`, @@ -54,6 +54,18 @@ module.exports.run = async function(options) { noop: options.noop, }); + await editRole({ + roleId: `${roleRoot}:push:${domain}:level-${level}:*`, + description: description( + `Scopes for tasks associated with pushes to ${domain} projects at level ${level}; + the '*' matches the project name.` + ), + scopes: [ + `in-tree:hook-action:project-${domain}/in-tree-action-${level}-*`, + ], + noop: options.noop, + }); + let makeFeature = async (feature, scopes) => { await editRole({ roleId: `${roleRoot}:feature:${feature}:${domain}:level-${level}:*`, diff --git a/src/make-scm-group-role.js b/src/make-scm-group-role.js deleted file mode 100644 index 11ab229..0000000 --- a/src/make-scm-group-role.js +++ /dev/null @@ -1,43 +0,0 @@ -const editRole = require('./util/edit-role'); -const {getProjects, hgmoPath} = require('./util/projects'); - -module.exports.setup = (program) => { - return program - .command('make-scm-group-role ') - .option('-n, --noop', 'Don\'t change roles, just show difference') - .description('create or update a mozilla-group:scm_foo role, based on production-branches.json'); -}; - -module.exports.run = async (group, options) => { - var taskcluster = require('taskcluster-client'); - var chalk = require('chalk'); - var auth = new taskcluster.Auth(); - - // find the list of projects with this group - var projects = await getProjects(); - var projectsWithGroup = Object.keys(projects) - .filter(p => projects[p].access === group) - .map(p => projects[p]); - - var roleId = 'mozilla-group:active_' + group; - var scopes = projectsWithGroup.map(project => { - let path = hgmoPath(project); - return `assume:repo:hg.mozilla.org/${path}:*`; - }); - - var description = [ - '*DO NOT EDIT*', - '', - 'Scopes for members of this group, based on repos with this access level', - '', - 'This role is configured automatically by [taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', - ].join('\n'); - - await editRole({ - roleId, - description, - scopes, - noop: options.noop, - }); -}; - diff --git a/src/make-scm-group-roles.js b/src/make-scm-group-roles.js new file mode 100644 index 0000000..4da6599 --- /dev/null +++ b/src/make-scm-group-roles.js @@ -0,0 +1,54 @@ +const editRole = require('./util/edit-role'); +const {ACTION_HOOKS} = require('./util/action-hooks'); +const {getProjects, hgmoPath} = require('./util/projects'); + +module.exports.setup = (program) => { + return program + .command('make-scm-group-roles') + .option('-n, --noop', 'Don\'t change roles, just show difference') + .description('create or update a mozilla-group:active_scm_level_[123] roles, based on ci configuration'); +}; + +module.exports.run = async (options) => { + var taskcluster = require('taskcluster-client'); + var chalk = require('chalk'); + var auth = new taskcluster.Auth(); + var projects = await getProjects(); + + for (let level of ['1', '2', '3']) { + // find the list of projects with this group + var projectsWithGroup = Object.keys(projects) + .filter(p => projects[p].access === `scm_level_${level}`) + .map(p => projects[p]); + + var roleId = `mozilla-group:active_scm_level_${level}`; + + // See https://bugzilla.mozilla.org/show_bug.cgi?id=1415868 for "old" and "new" + const oldScopes = projectsWithGroup.map(project => { + let path = hgmoPath(project); + return `assume:repo:hg.mozilla.org/${path}:*`; + }); + const newScopes = ACTION_HOOKS + .filter(ah => ah.level === level && ah.groups.includes(`active_scm_level_${level}`)) + .map(({trustDomain, actionPerm}) => + `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}`); + + const scopes = oldScopes.concat(newScopes); + + var description = [ + '*DO NOT EDIT*', + '', + 'Scopes for members of this group, allowing actions related to repos at this level.', + '', + 'This role is configured automatically by [taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', + ].join('\n'); + + await editRole({ + roleId, + description, + scopes, + noop: options.noop, + }); + } +}; + diff --git a/src/update-action-hook-perms.js b/src/update-action-hook-perms.js new file mode 100644 index 0000000..b8f72cb --- /dev/null +++ b/src/update-action-hook-perms.js @@ -0,0 +1,41 @@ +const editRole = require('./util/edit-role'); +const {ACTION_HOOKS} = require('./util/action-hooks'); + +module.exports.setup = (program) => { + return program + .command('update-action-hook-perms') + .option('-n, --noop', 'Don\'t change roles, just show difference') + .description('update action-related `hooks:trigger-hook:` scopes in mozilla-group:* roles'); +}; + +module.exports.run = async function(options) { + var taskcluster = require('taskcluster-client'); + var chalk = require('chalk'); + var _ = require('lodash'); + + var auth = new taskcluster.Auth(); + + const roles = await auth.listRoles(); + + for (let role of roles) { + const match = role.roleId.match(/^mozilla-group:(.*)$/); + if (!match) { + continue; + } + const group = match[1]; + + // find the expected actions + const expectedActions = ACTION_HOOKS.filter(ah => ah.groups.includes(group)); + const scopes = role.scopes + .filter(scope => !scope.match(/^hooks:trigger-hook:project-(gecko|comm)\/in-tree-action-/)) + .concat(expectedActions.map(({trustDomain, level, actionPerm}) => + `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}`)); + + await editRole({ + roleId: role.roleId, + description: role.description, + scopes, + noop: options.noop, + }); + } +}; diff --git a/src/util/action-hooks.js b/src/util/action-hooks.js new file mode 100644 index 0000000..e363ee9 --- /dev/null +++ b/src/util/action-hooks.js @@ -0,0 +1,24 @@ +exports.ACTION_HOOKS = [ + { + trustDomain: 'gecko', + level: '1', + actionPerm: 'generic', + groups: ['active_scm_level_1'], + // inputSchema, + }, + { + trustDomain: 'comm', + level: '1', + actionPerm: 'generic', + groups: ['active_scm_level_1'], + // inputSchema, + }, + { + trustDomain: 'gecko', + level: '1', + actionPerm: 'purge-caches', + groups: ['taskcluster', 'vpn_sheriff'], + // inputSchema, + }, +]; + diff --git a/src/util/edit-hook.js b/src/util/edit-hook.js new file mode 100644 index 0000000..15b3458 --- /dev/null +++ b/src/util/edit-hook.js @@ -0,0 +1,64 @@ +const chalk = require('chalk'); +const taskcluster = require('taskcluster-client'); +const {diffLines} = require('diff'); +const {showDiff} = require('./show-diff'); +const yaml = require('js-yaml'); + +const editHook = async ({hookGroupId, hookId, metadata, schedule, task, triggerSchema, noop}) => { + const newHook = { + metadata, + schedule, + task, + triggerSchema, + }; + + const hooks = new taskcluster.Hooks(); + let hook = {}; + try { + hook = await hooks.hook(hookGroupId, hookId); + delete hook['hookId']; + delete hook['hookGroupId']; + delete hook['expires']; + delete hook['deadline']; + } catch (err) { + if (err.statusCode !== 404) { + throw err; + } + } + + const diffs = diffLines( + yaml.safeDump(hook, {sortKeys: true, flowLevel: -1}), + yaml.safeDump(newHook, {sortKeys: true, flowLevel: -1})); + let diffsFound = false; + diffs.forEach(diff => { + if (diff.added || diff.removed) { + diffsFound = true; + } + }); + + if (diffsFound) { + console.log(chalk.green.bold(`changes required for hook ${hookGroupId}/${hookId}:`)); + showDiff({diffs, context: 8}); + } else { + console.log(chalk.green.bold(`no changes required for hook ${hookGroupId}/${hookId}`)); + } + + if (!noop && diffsFound) { + try { + if (hook.task) { + console.log(chalk.green.bold('updating hook')); + await hooks.updateHook(hookGroupId, hookId, newHook); + } else { + console.log(chalk.green.bold('creating hook')); + await hooks.createHook(hookGroupId, hookId, newHook); + } + } catch (err) { + console.log(err); + process.exit(1); + } + console.log(chalk.green.bold('done')); + } +}; + +module.exports = editHook; + diff --git a/src/util/edit-provisioner-worker-type.js b/src/util/edit-provisioner-worker-type.js index f71e508..e623987 100644 --- a/src/util/edit-provisioner-worker-type.js +++ b/src/util/edit-provisioner-worker-type.js @@ -2,23 +2,15 @@ const chalk = require('chalk'); const _ = require('lodash'); const taskcluster = require('taskcluster-client'); const {diffJson} = require('diff'); +const {showDiff} = require('./show-diff'); const editProvisionerWorkerType = async ({workerType, original, updated, noop}) => { const provisioner = new taskcluster.AwsProvisioner(); - var diff = diffJson(original, updated); - if (_.find(diff, {added: true}) || _.find(diff, {removed: true})) { + var diffs = diffJson(original, updated); + if (_.find(diffs, {added: true}) || _.find(diffs, {removed: true})) { console.log(chalk.green.bold(`changes required for workerType ${workerType}:`)); - diff.forEach(c => { - if (c.added) { - process.stdout.write(chalk.green(c.value)); - } else if (c.removed) { - process.stdout.write(chalk.red(c.value)); - } else { - process.stdout.write(c.value); - } - }); - process.stdout.write('\n'); + showDiff({diffs, context: 8}); } else { console.log(chalk.green.bold(`no changes required for workerType ${workerType}`)); return; diff --git a/src/util/show-diff.js b/src/util/show-diff.js new file mode 100644 index 0000000..4ab2e9c --- /dev/null +++ b/src/util/show-diff.js @@ -0,0 +1,31 @@ +const chalk = require('chalk'); + +/** + * Given the output from the 'diff' package, format it onscreen as a unified diff + * + * If context is set, lines more than this distance from another diff will not be + * shown. + */ +exports.showDiff = ({diffs, context}) => { + const lastIndex = diffs.length - 1; + diffs.forEach((diff, i) => { + const lines = diff.value.trimRight().split(/\n/); + const first = i === 0; + const last = i === lastIndex; + if (diff.added) { + lines.forEach(l => console.log(chalk.green('+' + l))); + } else if (diff.removed) { + lines.forEach(l => console.log(chalk.red('-' + l))); + } else if (context && lines.length > context * 2) { + if (!first) { + lines.slice(0, context).forEach(l => console.log(' ' + l)); + } + console.log(chalk.bold('...')); + if (!last) { + lines.slice(-context).forEach(l => console.log(' ' + l)); + } + } else { + lines.forEach(l => console.log(' ' + l)); + } + }); +}; diff --git a/src/util/tcyml.js b/src/util/tcyml.js new file mode 100644 index 0000000..295b49c --- /dev/null +++ b/src/util/tcyml.js @@ -0,0 +1,15 @@ +const request = require('superagent'); +const yaml = require('js-yaml'); + +const CENTRAL_RAW = 'https://hg.mozilla.org/mozilla-central/raw-file/default/'; + +/** + * Get the .taskcluster.yml for the given repository + */ +exports.getTaskclusterYml = async (repoPath) => { + let res = await request.get(`${repoPath}/raw-file/default/.taskcluster.yml`).buffer(true); + if (!res.ok) { + throw new Error(res.text); + } + return yaml.safeLoad(res.text); +};