From 7b9b45baaf5de170ecd695282d89962cff872c96 Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Tue, 22 May 2018 16:20:27 -0400 Subject: [PATCH 1/3] Bug 1463522 - hash .taskcluster.yml and use the result in the hook name --- src/make-gecko-action-hooks.js | 126 ++++++++++++++++++++------------ src/make-scm-group-roles.js | 2 +- src/update-action-hook-perms.js | 2 +- src/util/tcyml.js | 15 ---- 4 files changed, 83 insertions(+), 62 deletions(-) delete mode 100644 src/util/tcyml.js diff --git a/src/make-gecko-action-hooks.js b/src/make-gecko-action-hooks.js index 4b2d398..392c5bb 100644 --- a/src/make-gecko-action-hooks.js +++ b/src/make-gecko-action-hooks.js @@ -1,10 +1,13 @@ +const request = require('superagent'); +const yaml = require('js-yaml'); const _ = require('lodash'); +const crypto = require('crypto'); const {getProjects, hgmoPath, scmLevel} = require('./util/projects'); -const {getTaskclusterYml} = require('./util/tcyml'); const editRole = require('./util/edit-role'); const editHook = require('./util/edit-hook'); const {ACTION_HOOKS} = require('./util/action-hooks'); const chalk = require('chalk'); +const taskcluster = require('taskcluster-client'); module.exports.setup = (program) => { return program @@ -14,61 +17,94 @@ module.exports.setup = (program) => { }; module.exports.run = async function(options) { - let taskcluster = require('taskcluster-client'); - let chalk = require('chalk'); let projects = await getProjects(); - // We build action hooks' task definitions from the latest in-tree `.taskcluster.yml`. - const taskclusterYml = { - 'mozilla-central': await getTaskclusterYml(projects['mozilla-central'].repo), - 'comm-central': await getTaskclusterYml(projects['comm-central'].repo), - }; + const taskclusterYmls = await hashTaskclusterYmls(projects); + + for (let {taskclusterYmlHash, taskclusterYml} of taskclusterYmls) { + for (let action of ACTION_HOOKS) { + const hookGroupId = `project-${action.trustDomain}`; + const hookId = `in-tree-action-${action.level}-${action.actionPerm}_${taskclusterYmlHash}`; + const {task, triggerSchema} = makeHookDetails(taskclusterYml, action); + await editHook({ + noop: options.noop, + hookGroupId, + hookId, + metadata: { + name: [ + `Action task ${action.level}-${action.actionPerm}, with \`.taskcluster.yml\``, + `hash ${taskclusterYmlHash}`, + ].join(' '), + description: [ + '*DO NOT EDIT*', + '', + 'This hook is configured automatically by', + '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', + ].join('\n'), + owner: 'taskcluster-notifications@mozilla.com', + emailOnError: false, // true, TODO + }, + schedule: [], + task, + triggerSchema, + }); - for (let action of ACTION_HOOKS) { - const hookGroupId = `project-${action.trustDomain}`; - const hookId = `in-tree-action-${action.level}-${action.actionPerm}`; - const projName = action.trustDomain === 'gecko' ? 'mozilla-central' : 'comm-central'; - const {task, triggerSchema} = makeHookDetails(taskclusterYml[projName], action); - await editHook({ - noop: options.noop, - hookGroupId, - hookId, - metadata: { - name: `Action task ${action.level}-${action.actionPerm}`, + // make the role with scopes assume:repo::action: for each repo at this level + const projectsAtLevel = Object.keys(projects) + .filter(p => projects[p].access === `scm_level_${action.level}`) + .map(p => projects[p]); + const scopes = projectsAtLevel.map( + project => `assume:repo:hg.mozilla.org/${hgmoPath(project)}:action:${action.actionPerm}`); + await editRole({ + roleId: `hook-id:${hookGroupId}/${hookId}`, description: [ '*DO NOT EDIT*', '', - 'This hook is configured automatically by', + 'This role is configured automatically by', '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', ].join('\n'), - owner: 'taskcluster-notifications@mozilla.com', - emailOnError: false, // true, TODO - }, - schedule: [], - task, - triggerSchema, - }); - - // make the role with scopes assume:repo::action: for each repo at this level - const projectsAtLevel = Object.keys(projects) - .filter(p => projects[p].access === `scm_level_${action.level}`) - .map(p => projects[p]); - const scopes = projectsAtLevel.map( - project => `assume:repo:hg.mozilla.org/${hgmoPath(project)}:action:${action.actionPerm}`); - await editRole({ - roleId: `hook-id:${hookGroupId}/${hookId}`, - description: [ - '*DO NOT EDIT*', - '', - 'This role is configured automatically by', - '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).', - ].join('\n'), - scopes, - noop: options.noop, - }); + scopes, + noop: options.noop, + }); + } } }; +const hashTaskclusterYmls = async (projects) => { + console.log(chalk.yellow.bold('fetching and hashing `.taskcluster.yml` files from all repos')); + + const result = {}; + await Promise.all(Object.entries(projects).map(async ([alias, {repo, features}]) => { + // try repos don't have a `default` so there's nothing to do + if (alias.startsWith('try') || alias.endsWith('try')) { + return; + } + + let taskclusterYml; + try { + const res = await request.get(`${repo}/raw-file/default/.taskcluster.yml`).buffer(true); + taskclusterYml = res.text; + } catch (err) { + // if no .taskcluster.yml exists, skip it.. + if (err.status === 404) { + return; + } + } + + const taskclusterYmlHash = crypto + .createHash('sha256') + .update(taskclusterYml) + .digest('hex') + .slice(0, 10); + + result[taskclusterYmlHash] = yaml.safeLoad(taskclusterYml); + })); + + return Object + .entries(result) + .map(([taskclusterYmlHash, taskclusterYml]) => ({taskclusterYmlHash, taskclusterYml})); +}; + const makeHookDetails = (taskclusterYml, action) => { const task = { $let: { diff --git a/src/make-scm-group-roles.js b/src/make-scm-group-roles.js index 4da6599..513973e 100644 --- a/src/make-scm-group-roles.js +++ b/src/make-scm-group-roles.js @@ -31,7 +31,7 @@ module.exports.run = async (options) => { const newScopes = ACTION_HOOKS .filter(ah => ah.level === level && ah.groups.includes(`active_scm_level_${level}`)) .map(({trustDomain, actionPerm}) => - `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}`); + `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}_*`); const scopes = oldScopes.concat(newScopes); diff --git a/src/update-action-hook-perms.js b/src/update-action-hook-perms.js index b8f72cb..421772d 100644 --- a/src/update-action-hook-perms.js +++ b/src/update-action-hook-perms.js @@ -29,7 +29,7 @@ module.exports.run = async function(options) { const scopes = role.scopes .filter(scope => !scope.match(/^hooks:trigger-hook:project-(gecko|comm)\/in-tree-action-/)) .concat(expectedActions.map(({trustDomain, level, actionPerm}) => - `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}`)); + `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}_*`)); await editRole({ roleId: role.roleId, diff --git a/src/util/tcyml.js b/src/util/tcyml.js deleted file mode 100644 index 295b49c..0000000 --- a/src/util/tcyml.js +++ /dev/null @@ -1,15 +0,0 @@ -const request = require('superagent'); -const yaml = require('js-yaml'); - -const CENTRAL_RAW = 'https://hg.mozilla.org/mozilla-central/raw-file/default/'; - -/** - * Get the .taskcluster.yml for the given repository - */ -exports.getTaskclusterYml = async (repoPath) => { - let res = await request.get(`${repoPath}/raw-file/default/.taskcluster.yml`).buffer(true); - if (!res.ok) { - throw new Error(res.text); - } - return yaml.safeLoad(res.text); -}; From c1a08ac6496af26d082ec06a2afed5c71b93acfa Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Tue, 22 May 2018 16:45:30 -0400 Subject: [PATCH 2/3] Bug 1463522 - color the colon differently to avoid confusion --- src/util/edit-hook.js | 2 +- src/util/edit-provisioner-worker-type.js | 2 +- src/util/edit-role.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/util/edit-hook.js b/src/util/edit-hook.js index 15b3458..207bf8b 100644 --- a/src/util/edit-hook.js +++ b/src/util/edit-hook.js @@ -37,7 +37,7 @@ const editHook = async ({hookGroupId, hookId, metadata, schedule, task, triggerS }); if (diffsFound) { - console.log(chalk.green.bold(`changes required for hook ${hookGroupId}/${hookId}:`)); + console.log(chalk.green.bold(`changes required for hook ${hookGroupId}/${hookId}`) + ':'); showDiff({diffs, context: 8}); } else { console.log(chalk.green.bold(`no changes required for hook ${hookGroupId}/${hookId}`)); diff --git a/src/util/edit-provisioner-worker-type.js b/src/util/edit-provisioner-worker-type.js index e623987..1e9fd8d 100644 --- a/src/util/edit-provisioner-worker-type.js +++ b/src/util/edit-provisioner-worker-type.js @@ -9,7 +9,7 @@ const editProvisionerWorkerType = async ({workerType, original, updated, noop}) var diffs = diffJson(original, updated); if (_.find(diffs, {added: true}) || _.find(diffs, {removed: true})) { - console.log(chalk.green.bold(`changes required for workerType ${workerType}:`)); + console.log(chalk.green.bold(`changes required for workerType ${workerType}`) + ':'); showDiff({diffs, context: 8}); } else { console.log(chalk.green.bold(`no changes required for workerType ${workerType}`)); diff --git a/src/util/edit-role.js b/src/util/edit-role.js index d3766e4..9077b20 100644 --- a/src/util/edit-role.js +++ b/src/util/edit-role.js @@ -17,7 +17,7 @@ const editRole = async ({roleId, description, scopes, noop}) => { var got_scopes = role ? role.scopes : []; var diff = arrayDiff(got_scopes, scopes); if (diff.removed.length || diff.added.length) { - console.log(chalk.green.bold(`scope changes required for role ${roleId}:`)); + console.log(chalk.green.bold(`scope changes required for role ${roleId}`) + ':'); diff.removed.forEach(s => console.log(chalk.red(`- ${s}`))); diff.added.forEach(s => console.log(chalk.green(`+ ${s}`))); } else { From 591ae460d3bd1211905d9a14d8076975a25994ff Mon Sep 17 00:00:00 2001 From: "Dustin J. Mitchell" Date: Tue, 22 May 2018 16:50:14 -0400 Subject: [PATCH 3/3] Bug 1463522 - use project-

:in-tree-action-trigger: for trigger scopes This avoids the need to do any role "merging" wher a role is both manually edited and merged by tcadmin. --- src/make-scm-group-roles.js | 14 ++++----- src/update-action-hook-perms.js | 52 +++++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/src/make-scm-group-roles.js b/src/make-scm-group-roles.js index 513973e..2865f87 100644 --- a/src/make-scm-group-roles.js +++ b/src/make-scm-group-roles.js @@ -23,17 +23,13 @@ module.exports.run = async (options) => { var roleId = `mozilla-group:active_scm_level_${level}`; - // See https://bugzilla.mozilla.org/show_bug.cgi?id=1415868 for "old" and "new" - const oldScopes = projectsWithGroup.map(project => { + const scopes = projectsWithGroup.map(project => { let path = hgmoPath(project); return `assume:repo:hg.mozilla.org/${path}:*`; - }); - const newScopes = ACTION_HOOKS - .filter(ah => ah.level === level && ah.groups.includes(`active_scm_level_${level}`)) - .map(({trustDomain, actionPerm}) => - `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}_*`); - - const scopes = oldScopes.concat(newScopes); + }).concat([ + `assume:project-gecko:in-tree-action-trigger:active_scm_level_${level}`, + `assume:project-comm:in-tree-action-trigger:active_scm_level_${level}`, + ]); var description = [ '*DO NOT EDIT*', diff --git a/src/update-action-hook-perms.js b/src/update-action-hook-perms.js index 421772d..c4352c7 100644 --- a/src/update-action-hook-perms.js +++ b/src/update-action-hook-perms.js @@ -1,39 +1,59 @@ const editRole = require('./util/edit-role'); const {ACTION_HOOKS} = require('./util/action-hooks'); +const taskcluster = require('taskcluster-client'); +const chalk = require('chalk'); +const _ = require('lodash'); module.exports.setup = (program) => { return program .command('update-action-hook-perms') .option('-n, --noop', 'Don\'t change roles, just show difference') - .description('update action-related `hooks:trigger-hook:` scopes in mozilla-group:* roles'); + .description([ + 'Update action-related `hooks:trigger-hook:` scopes in `project-{gecko,comm}:in-tree-action-trigger:`', + 'roles', + ].join('\n')); }; module.exports.run = async function(options) { - var taskcluster = require('taskcluster-client'); - var chalk = require('chalk'); - var _ = require('lodash'); - - var auth = new taskcluster.Auth(); + const auth = new taskcluster.Auth(); + let toEdit = []; + // enumerate all maching roles (so we delete things as necessary) const roles = await auth.listRoles(); - for (let role of roles) { - const match = role.roleId.match(/^mozilla-group:(.*)$/); + const match = role.roleId.match(/^project-(gecko|comm):in-tree-action-trigger:(.*)$/); if (!match) { continue; } - const group = match[1]; + const trustDomain = match[2]; + const group = match[2]; + toEdit.push([trustDomain, group]); + } + // and enumerate the expected roles + for (let {trustDomain, groups} of ACTION_HOOKS) { + for (let group of groups) { + toEdit.push([trustDomain, group]); + } + } + + toEdit = _.uniqBy(toEdit, ([trustDomain, group]) => `${trustDomain}:${group}`); + for (let [trustDomain, group] of toEdit) { // find the expected actions - const expectedActions = ACTION_HOOKS.filter(ah => ah.groups.includes(group)); - const scopes = role.scopes - .filter(scope => !scope.match(/^hooks:trigger-hook:project-(gecko|comm)\/in-tree-action-/)) - .concat(expectedActions.map(({trustDomain, level, actionPerm}) => - `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}_*`)); + const expectedActions = ACTION_HOOKS.filter( + ah => ah.groups.includes(group) && ah.trustDomain === trustDomain); + const scopes = expectedActions.map(({trustDomain, level, actionPerm}) => + `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}_*`); await editRole({ - roleId: role.roleId, - description: role.description, + roleId: `project-${trustDomain}:in-tree-action-trigger:${group}`, + description: [ + '*DO NOT EDIT*', + '', + `Permissions to trigger ${trustDomain}-related hooks for ${group}. This role should`, + `be assumed by \`mozilla-group:${group}\`.`, + '', + ].join('\n'), scopes, noop: options.noop, });