Add Bindings view in Hooks create page

infrastructure/terraform/web-ui.tf

@@ -4,7 +4,6 @@ module "web_ui_secrets" {
 
   secrets = {
     APPLICATION_NAME              = "Taskcluster"
-    PORT                          = ""
     GRAPHQL_ENDPOINT              = "${var.root_url}/graphql"
     GRAPHQL_SUBSCRIPTION_ENDPOINT = "${var.root_url}/subscription"
 

libraries/testing/src/time.js

@@ -25,7 +25,6 @@ exports.runWithFakeTime = (fn, {mock=true, maxTime=30000, ...zurvanOptions}={})
     });
 
     let finished, err;
-    this.slow(maxTime);
     fn.apply(this, []).then(
       () => {
         finished = true;

services/web-server/src/graphql/Hooks.graphql

@@ -51,6 +51,7 @@ type HookStatus {
 type Hook {
   hookGroupId: ID!
   hookId: ID!
+  bindings: [HookBinding]
   metadata: HookMetadata!
   schedule: [String]!
   task: JSON!
@@ -72,11 +73,22 @@ type DeleteHook {
   hookId: ID
 }
 
+type HookBinding {
+  exchange: String!
+  routingKeyPattern: String!
+}
+
+input HookBindingInput {
+  exchange: String!
+  routingKeyPattern: String!
+}
+
 input HookInput {
   metadata: HookMetadataInput!
   schedule: [String]
   expires: String
   deadline: String
+  bindings: [HookBindingInput]
   task: JSON!
   triggerSchema: JSON
 }

services/worker-manager/src/data.js

@@ -104,7 +104,7 @@ WorkerPool.expire = async function(monitor) {
   });
 };
 
-WorkerPool.prototype.reportError = async function({kind, title, description, extra={}, notify}) {
+WorkerPool.prototype.reportError = async function({kind, title, description, extra={}, notify, WorkerPoolError}) {
   if (this.emailOnError) {
     let extraInfo = '';
     if (Object.keys(extra).length) {

services/worker-manager/src/main.js

@@ -182,10 +182,10 @@ let load = loader({
   },
 
   providers: {
-    requires: ['cfg', 'monitor', 'notify', 'estimator', 'Worker', 'WorkerPool', 'schemaset'],
-    setup: async ({cfg, monitor, notify, estimator, Worker, WorkerPool, schemaset}) =>
+    requires: ['cfg', 'monitor', 'notify', 'estimator', 'Worker', 'WorkerPool', 'WorkerPoolError', 'schemaset'],
+    setup: async ({cfg, monitor, notify, estimator, Worker, WorkerPool, WorkerPoolError, schemaset}) =>
       new Providers().setup({
-        cfg, monitor, notify, estimator, Worker, WorkerPool,
+        cfg, monitor, notify, estimator, Worker, WorkerPool, WorkerPoolError,
         validator: await schemaset.validator(cfg.taskcluster.rootUrl),
       }),
   },

services/worker-manager/src/providers/google.js

@@ -24,6 +24,7 @@ class GoogleProvider extends Provider {
     validator,
     Worker,
     WorkerPool,
+    WorkerPoolError,
     fake = false,
   }) {
     super({
@@ -33,9 +34,10 @@ class GoogleProvider extends Provider {
       notify,
       rootUrl,
       estimator,
-      Worker,
       validator,
+      Worker,
       WorkerPool,
+      WorkerPoolError,
     });
     this.configSchema = 'config-google';
     this.fake = fake;
@@ -374,6 +376,7 @@ class GoogleProvider extends Provider {
             title: 'Instance Creation Error',
             description: error.message, // TODO: Make sure we clear exposing this with security folks
             notify: this.notify,
+            WorkerPoolError: this.WorkerPoolError,
           });
         }
         return;
@@ -481,6 +484,7 @@ class GoogleProvider extends Provider {
             code: err.code,
           },
           notify: this.notify,
+          WorkerPoolError: this.WorkerPoolError,
         });
       }
     }

services/worker-manager/src/providers/index.js

@@ -3,7 +3,7 @@
  * their providerType implementation as required
  */
 class Providers {
-  async setup({cfg, monitor, notify, estimator, Worker, WorkerPool, validator}) {
+  async setup({cfg, monitor, notify, estimator, Worker, WorkerPool, WorkerPoolError, validator}) {
     this._providers = {};
 
     if (cfg.providers['null-provider']) {
@@ -26,12 +26,13 @@ class Providers {
       const provider = new Provider({
         providerId,
         notify,
-        monitor: monitor.childMonitor(providerId),
+        monitor: monitor.childMonitor(`provider.${providerId}`),
         rootUrl: cfg.taskcluster.rootUrl,
         taskclusterCredentials: cfg.taskcluster.credentials,
         estimator,
         Worker,
         WorkerPool,
+        WorkerPoolError,
         validator,
         ...meta,
       });

services/worker-manager/src/providers/provider.js

@@ -16,9 +16,10 @@ class Provider {
     rootUrl,
     taskclusterCredentials,
     estimator,
-    Worker,
     validator,
+    Worker,
     WorkerPool,
+    WorkerPoolError,
   }) {
     this.providerId = providerId;
     this.monitor = monitor;
@@ -29,6 +30,7 @@ class Provider {
     this.estimator = estimator;
     this.Worker = Worker;
     this.WorkerPool = WorkerPool;
+    this.WorkerPoolError = WorkerPoolError;
   }
 
   /**

services/worker-manager/src/providers/testing.js

@@ -17,6 +17,24 @@ class TestingProvider extends Provider {
   async removeResources({workerPool}) {
     this.monitor.notice('remove-resource', {workerPoolId: workerPool.workerPoolId});
   }
+
+  async provision({workerPool}) {
+    this.monitor.notice('test-provision', {workerPoolId: workerPool.workerPoolId});
+  }
+
+  async scanPrepare() {
+    this.monitor.notice('scan-prepare', {});
+  }
+
+  async checkWorker({worker}) {
+    await worker.modify(w => {
+      w.providerData.checked = true;
+    });
+  }
+
+  async scanCleanup() {
+    this.monitor.notice('scan-cleanup', {});
+  }
 }
 
 module.exports = {

services/worker-manager/src/worker-scanner.js

@@ -41,16 +41,16 @@ class WorkerScanner {
   }
 
   async scan() {
-    await Promise.all(Object.values(this.providers).map(x => x.scanPrepare()));
+    await this.providers.forAll(p => p.scanPrepare());
     await this.Worker.scan({
       state: Entity.op.notEqual(this.Worker.states.STOPPED),
     }, {
       handler: async worker => {
-        const provider = this.providers[worker.providerId];
+        const provider = this.providers.get(worker.providerId);
         await provider.checkWorker({worker});
       },
     });
-    await Promise.all(Object.values(this.providers).map(x => x.scanCleanup()));
+    await this.providers.forAll(p => p.scanCleanup());
   }
 }
 

services/worker-manager/test/helper.js

@@ -69,6 +69,34 @@ exports.withProvisioner = (mock, skipping) => {
   });
 };
 
+exports.withWorkerScanner = (mock, skipping) => {
+  let scanner;
+
+  suiteSetup(async function() {
+    if (skipping()) {
+      return;
+    }
+    exports.initiateWorkerScanner = async () => {
+      scanner = await exports.load('workerScanner');
+      // remove it right away, as it is started on load
+      exports.load.remove('workerScanner');
+      return scanner;
+    };
+    exports.terminateWorkerScanner = async () => {
+      if (scanner) {
+        await scanner.terminate();
+        scanner = null;
+      }
+    };
+  });
+
+  teardown(function() {
+    if (scanner) {
+      throw new Error('Must call terminateWorkerScanner if you have started it');
+    }
+  });
+};
+
 /**
  * Set up a fake tc-queue object that supports only the `pendingTasks` method,
  * and inject that into the loader.  This is injected regardless of

services/worker-manager/test/provisioner_test.js

@@ -37,6 +37,14 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'azure'], function
                 Fields: {workerPoolId: wt.workerPoolId, providerId: wt.input.providerId, v: 1},
                 Severity: LEVELS.info,
               });
+            assert.deepEqual(
+              monitorManager.messages.find(
+                msg => msg.Type === 'test-provision' && msg.Fields.workerPoolId === wt.workerPoolId), {
+                Logger: `taskcluster.worker-manager.provider.${wt.input.providerId}`,
+                Type: 'test-provision',
+                Fields: {workerPoolId: wt.workerPoolId},
+                Severity: LEVELS.notice,
+              });
           }));
         });
         await helper.terminateProvisioner();
@@ -144,7 +152,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'azure'], function
         routes: [],
       });
       assert.deepEqual(monitorManager.messages.find(msg => msg.Type === 'create-resource'), {
-        Logger: 'taskcluster.worker-manager.testing1',
+        Logger: 'taskcluster.worker-manager.provider.testing1',
         Type: 'create-resource',
         Severity: LEVELS.notice,
         Fields: {workerPoolId: 'pp/foo'},
@@ -176,7 +184,7 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'azure'], function
         routes: [],
       });
       assert.deepEqual(monitorManager.messages.find(msg => msg.Type === 'update-resource'), {
-        Logger: 'taskcluster.worker-manager.testing1',
+        Logger: 'taskcluster.worker-manager.provider.testing1',
         Type: 'update-resource',
         Severity: LEVELS.notice,
         Fields: {workerPoolId: 'pp/foo'},
@@ -198,13 +206,13 @@ helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'azure'], function
         routes: [],
       });
       assert.deepEqual(monitorManager.messages.find(msg => msg.Type === 'remove-resource'), {
-        Logger: 'taskcluster.worker-manager.testing1',
+        Logger: 'taskcluster.worker-manager.provider.testing1',
         Type: 'remove-resource',
         Severity: LEVELS.notice,
         Fields: {workerPoolId: 'pp/foo'},
       });
       assert.deepEqual(monitorManager.messages.find(msg => msg.Type === 'create-resource'), {
-        Logger: 'taskcluster.worker-manager.testing2',
+        Logger: 'taskcluster.worker-manager.provider.testing2',
         Type: 'create-resource',
         Severity: LEVELS.notice,
         Fields: {workerPoolId: 'pp/foo'},

services/worker-manager/test/worker_pool_errors.js

@@ -0,0 +1,54 @@
+const slugid = require('slugid');
+const assert = require('assert');
+const helper = require('./helper');
+const testing = require('taskcluster-lib-testing');
+
+helper.secrets.mockSuite(testing.suiteName(), ['taskcluster', 'azure'], function(mock, skipping) {
+  helper.withEntities(mock, skipping);
+  helper.withPulse(mock, skipping);
+  helper.withFakeNotify(mock, skipping);
+
+  test('create worker pool error', async function() {
+    await helper.WorkerPoolError.create({
+      workerPoolId: 'baz/foo',
+      errorId: slugid.v4(),
+      reported: new Date(),
+      kind: 'abc',
+      title: 'ABC',
+      description: 'ay bee see',
+      extra: {},
+    });
+
+    const errors = await helper.WorkerPoolError.scan({}, {});
+    assert.equal(errors.entries.length, 1);
+  });
+
+  test('report errors (no email)', async function() {
+    const wp = await helper.WorkerPool.create({
+      workerPoolId: 'ww/tt',
+      providerId: 'testing1',
+      description: 'none',
+      previousProviderIds: [],
+      scheduledForDeletion: false,
+      created: new Date(),
+      lastModified: new Date(),
+      config: {
+      },
+      owner: 'whatever@example.com',
+      providerData: {},
+      emailOnError: false,
+    });
+
+    await wp.reportError({
+      kind: 'something-error',
+      title: 'And Error about Something',
+      description: 'WHO KNOWS',
+      notify: helper.notify,
+      WorkerPoolError: helper.WorkerPoolError,
+    });
+
+    const errors = await helper.WorkerPoolError.scan({}, {});
+    assert.equal(errors.entries.length, 1);
+    assert.equal(errors.entries[0].workerPoolId, 'ww/tt');
+  });
+});