Bug 1463522 - use taskcluster.yml hashes

src/make-gecko-action-hooks.js

@@ -1,10 +1,13 @@
+const request = require('superagent');
+const yaml = require('js-yaml');
 const _ = require('lodash');
+const crypto = require('crypto');
 const {getProjects, hgmoPath, scmLevel} = require('./util/projects');
-const {getTaskclusterYml} = require('./util/tcyml');
 const editRole = require('./util/edit-role');
 const editHook = require('./util/edit-hook');
 const {ACTION_HOOKS} = require('./util/action-hooks');
 const chalk = require('chalk');
+const taskcluster = require('taskcluster-client');
 
 module.exports.setup = (program) => {
   return program
@@ -14,61 +17,94 @@ module.exports.setup = (program) => {
 };
 
 module.exports.run = async function(options) {
-  let taskcluster = require('taskcluster-client');
-  let chalk = require('chalk');
   let projects = await getProjects();
 
-  // We build action hooks' task definitions from the latest in-tree `.taskcluster.yml`.
-  const taskclusterYml = {
-    'mozilla-central': await getTaskclusterYml(projects['mozilla-central'].repo),
-    'comm-central': await getTaskclusterYml(projects['comm-central'].repo),
-  };
+  const taskclusterYmls = await hashTaskclusterYmls(projects);
+
+  for (let {taskclusterYmlHash, taskclusterYml} of taskclusterYmls) {
+    for (let action of ACTION_HOOKS) {
+      const hookGroupId = `project-${action.trustDomain}`;
+      const hookId = `in-tree-action-${action.level}-${action.actionPerm}_${taskclusterYmlHash}`;
+      const {task, triggerSchema} = makeHookDetails(taskclusterYml, action);
+      await editHook({
+        noop: options.noop,
+        hookGroupId,
+        hookId,
+        metadata: {
+          name: [
+            `Action task ${action.level}-${action.actionPerm}, with \`.taskcluster.yml\``,
+            `hash ${taskclusterYmlHash}`,
+          ].join(' '),
+          description: [
+            '*DO NOT EDIT*',
+            '',
+            'This hook is configured automatically by',
+            '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).',
+          ].join('\n'),
+          owner: 'taskcluster-notifications@mozilla.com',
+          emailOnError: false, // true, TODO
+        },
+        schedule: [],
+        task,
+        triggerSchema,
+      });
 
-  for (let action of ACTION_HOOKS) {
-    const hookGroupId = `project-${action.trustDomain}`;
-    const hookId = `in-tree-action-${action.level}-${action.actionPerm}`;
-    const projName = action.trustDomain === 'gecko' ? 'mozilla-central' : 'comm-central';
-    const {task, triggerSchema} = makeHookDetails(taskclusterYml[projName], action);
-    await editHook({
-      noop: options.noop,
-      hookGroupId,
-      hookId,
-      metadata: {
-        name: `Action task ${action.level}-${action.actionPerm}`,
+      // make the role with scopes assume:repo:<repo>:action:<actionPerm> for each repo at this level
+      const projectsAtLevel = Object.keys(projects)
+        .filter(p => projects[p].access === `scm_level_${action.level}`)
+        .map(p => projects[p]);
+      const scopes = projectsAtLevel.map(
+        project => `assume:repo:hg.mozilla.org/${hgmoPath(project)}:action:${action.actionPerm}`);
+      await editRole({
+        roleId: `hook-id:${hookGroupId}/${hookId}`,
         description: [
           '*DO NOT EDIT*',
           '',
-          'This hook is configured automatically by',
+          'This role is configured automatically by',
           '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).',
         ].join('\n'),
-        owner: 'taskcluster-notifications@mozilla.com',
-        emailOnError: false, // true, TODO
-      },
-      schedule: [],
-      task,
-      triggerSchema,
-    });
-
-    // make the role with scopes assume:repo:<repo>:action:<actionPerm> for each repo at this level
-    const projectsAtLevel = Object.keys(projects)
-      .filter(p => projects[p].access === `scm_level_${action.level}`)
-      .map(p => projects[p]);
-    const scopes = projectsAtLevel.map(
-      project => `assume:repo:hg.mozilla.org/${hgmoPath(project)}:action:${action.actionPerm}`);
-    await editRole({
-      roleId: `hook-id:${hookGroupId}/${hookId}`,
-      description: [
-        '*DO NOT EDIT*',
-        '',
-        'This role is configured automatically by',
-        '[taskcluster-admin](https://github.com/taskcluster/taskcluster-admin).',
-      ].join('\n'),
-      scopes,
-      noop: options.noop,
-    });
+        scopes,
+        noop: options.noop,
+      });
+    }
   }
 };
 
+const hashTaskclusterYmls = async (projects) => {
+  console.log(chalk.yellow.bold('fetching and hashing `.taskcluster.yml` files from all repos'));
+
+  const result = {};
+  await Promise.all(Object.entries(projects).map(async ([alias, {repo, features}]) => {
+    // try repos don't have a `default` so there's nothing to do
+    if (alias.startsWith('try') || alias.endsWith('try')) {
+      return;
+    }
+
+    let taskclusterYml;
+    try {
+      const res = await request.get(`${repo}/raw-file/default/.taskcluster.yml`).buffer(true);
+      taskclusterYml = res.text;
+    } catch (err) {
+      // if no .taskcluster.yml exists, skip it..
+      if (err.status === 404) {
+        return;
+      }
+    }
+
+    const taskclusterYmlHash = crypto
+      .createHash('sha256')
+      .update(taskclusterYml)
+      .digest('hex')
+      .slice(0, 10);
+
+    result[taskclusterYmlHash] = yaml.safeLoad(taskclusterYml);
+  }));
+
+  return Object
+    .entries(result)
+    .map(([taskclusterYmlHash, taskclusterYml]) => ({taskclusterYmlHash, taskclusterYml}));
+};
+
 const makeHookDetails = (taskclusterYml, action) => {
   const task = {
     $let: {

src/make-scm-group-roles.js

@@ -23,17 +23,13 @@ module.exports.run = async (options) => {
 
     var roleId = `mozilla-group:active_scm_level_${level}`;
 
-    // See https://bugzilla.mozilla.org/show_bug.cgi?id=1415868 for "old" and "new"
-    const oldScopes = projectsWithGroup.map(project => {
+    const scopes = projectsWithGroup.map(project => {
       let path = hgmoPath(project);
       return `assume:repo:hg.mozilla.org/${path}:*`;
-    });
-    const newScopes = ACTION_HOOKS
-      .filter(ah => ah.level === level && ah.groups.includes(`active_scm_level_${level}`))
-      .map(({trustDomain, actionPerm}) =>
-        `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}`);
-
-    const scopes = oldScopes.concat(newScopes);
+    }).concat([
+      `assume:project-gecko:in-tree-action-trigger:active_scm_level_${level}`,
+      `assume:project-comm:in-tree-action-trigger:active_scm_level_${level}`,
+    ]);
 
     var description = [
       '*DO NOT EDIT*',

src/update-action-hook-perms.js

@@ -1,39 +1,59 @@
 const editRole = require('./util/edit-role');
 const {ACTION_HOOKS} = require('./util/action-hooks');
+const taskcluster = require('taskcluster-client');
+const chalk = require('chalk');
+const _ = require('lodash');
 
 module.exports.setup = (program) => {
   return program
     .command('update-action-hook-perms')
     .option('-n, --noop', 'Don\'t change roles, just show difference')
-    .description('update action-related `hooks:trigger-hook:` scopes in mozilla-group:* roles');
+    .description([
+      'Update action-related `hooks:trigger-hook:` scopes in `project-{gecko,comm}:in-tree-action-trigger:`',
+      'roles',
+    ].join('\n'));
 };
 
 module.exports.run = async function(options) {
-  var taskcluster = require('taskcluster-client');
-  var chalk = require('chalk');
-  var _ = require('lodash');
-
-  var auth = new taskcluster.Auth();
+  const auth = new taskcluster.Auth();
+  let toEdit = [];
 
+  // enumerate all maching roles (so we delete things as necessary)
   const roles = await auth.listRoles();
-
   for (let role of roles) {
-    const match = role.roleId.match(/^mozilla-group:(.*)$/);
+    const match = role.roleId.match(/^project-(gecko|comm):in-tree-action-trigger:(.*)$/);
     if (!match) {
       continue;
     }
-    const group = match[1];
+    const trustDomain = match[2];
+    const group = match[2];
+    toEdit.push([trustDomain, group]);
+  }
 
+  // and enumerate the expected roles
+  for (let {trustDomain, groups} of ACTION_HOOKS) {
+    for (let group of groups) {
+      toEdit.push([trustDomain, group]);
+    }
+  }
+
+  toEdit = _.uniqBy(toEdit, ([trustDomain, group]) => `${trustDomain}:${group}`);
+  for (let [trustDomain, group] of toEdit) {
     // find the expected actions
-    const expectedActions = ACTION_HOOKS.filter(ah => ah.groups.includes(group));
-    const scopes = role.scopes
-      .filter(scope => !scope.match(/^hooks:trigger-hook:project-(gecko|comm)\/in-tree-action-/))
-      .concat(expectedActions.map(({trustDomain, level, actionPerm}) =>
-        `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}`));
+    const expectedActions = ACTION_HOOKS.filter(
+      ah => ah.groups.includes(group) && ah.trustDomain === trustDomain);
+    const scopes = expectedActions.map(({trustDomain, level, actionPerm}) =>
+      `hooks:trigger-hook:project-${trustDomain}/in-tree-action-${level}-${actionPerm}_*`);
 
     await editRole({
-      roleId: role.roleId,
-      description: role.description,
+      roleId: `project-${trustDomain}:in-tree-action-trigger:${group}`,
+      description: [
+        '*DO NOT EDIT*',
+        '',
+        `Permissions to trigger ${trustDomain}-related hooks for ${group}. This role should`,
+        `be assumed by \`mozilla-group:${group}\`.`,
+        '',
+      ].join('\n'),
       scopes,
       noop: options.noop,
     });

src/util/edit-hook.js

@@ -37,7 +37,7 @@ const editHook = async ({hookGroupId, hookId, metadata, schedule, task, triggerS
   });
 
   if (diffsFound) {
-    console.log(chalk.green.bold(`changes required for hook ${hookGroupId}/${hookId}:`));
+    console.log(chalk.green.bold(`changes required for hook ${hookGroupId}/${hookId}`) + ':');
     showDiff({diffs, context: 8});
   } else {
     console.log(chalk.green.bold(`no changes required for hook ${hookGroupId}/${hookId}`));

src/util/edit-provisioner-worker-type.js

@@ -9,7 +9,7 @@ const editProvisionerWorkerType = async ({workerType, original, updated, noop})
 
   var diffs = diffJson(original, updated);
   if (_.find(diffs, {added: true}) || _.find(diffs, {removed: true})) {
-    console.log(chalk.green.bold(`changes required for workerType ${workerType}:`));
+    console.log(chalk.green.bold(`changes required for workerType ${workerType}`) + ':');
     showDiff({diffs, context: 8});
   } else {
     console.log(chalk.green.bold(`no changes required for workerType ${workerType}`));

src/util/edit-role.js

@@ -17,7 +17,7 @@ const editRole = async ({roleId, description, scopes, noop}) => {
   var got_scopes = role ? role.scopes : [];
   var diff = arrayDiff(got_scopes, scopes);
   if (diff.removed.length || diff.added.length) {
-    console.log(chalk.green.bold(`scope changes required for role ${roleId}:`));
+    console.log(chalk.green.bold(`scope changes required for role ${roleId}`) + ':');
     diff.removed.forEach(s => console.log(chalk.red(`- ${s}`)));
     diff.added.forEach(s => console.log(chalk.green(`+ ${s}`)));
   } else {