From a9eb75e67f9848a00aed1d8a68029d22d466742b Mon Sep 17 00:00:00 2001
From: Patrick Mueller <pmuellr@gmail.com>
Date: Tue, 11 Feb 2020 18:20:56 -0500
Subject: [PATCH] instrument task manager with apm transactions (#55356)

Adds some apm transaction boundaries for parts of task manager, so that
they will show up in APM as new types of transactions.  Should provide
some visibility into the ES calls made by task manager for alerting and
actions, especially under stress testing scenarios.
---
 x-pack/plugins/task_manager/server/task_runner.ts | 14 ++++++++++++++
 x-pack/plugins/task_manager/server/task_store.ts  |  3 +++
 2 files changed, 17 insertions(+)

diff --git a/x-pack/plugins/task_manager/server/task_runner.ts b/x-pack/plugins/task_manager/server/task_runner.ts
index 56ab49bdc629ea..682885aaa0b1ca 100644
--- a/x-pack/plugins/task_manager/server/task_runner.ts
+++ b/x-pack/plugins/task_manager/server/task_runner.ts
@@ -10,6 +10,7 @@
  * rescheduling, middleware application, etc.
  */
 
+import apm from 'elastic-apm-node';
 import { performance } from 'perf_hooks';
 import Joi from 'joi';
 import { identity, defaults, flow } from 'lodash';
@@ -156,15 +157,21 @@ export class TaskManagerRunner implements TaskRunner {
       taskInstance: this.instance,
     });
 
+    const apmTrans = apm.startTransaction(
+      `taskManager run ${this.instance.taskType}`,
+      'taskManager'
+    );
     try {
       this.task = this.definition.createTaskRunner(modifiedContext);
       const result = await this.task.run();
       const validatedResult = this.validateResult(result);
+      if (apmTrans) apmTrans.end('success');
       return this.processResult(validatedResult);
     } catch (err) {
       this.logger.error(`Task ${this} failed: ${err}`);
       // in error scenario, we can not get the RunResult
       // re-use modifiedContext's state, which is correct as of beforeRun
+      if (apmTrans) apmTrans.end('error');
       return this.processResult(asErr({ error: err, state: modifiedContext.taskInstance.state }));
     }
   }
@@ -178,6 +185,11 @@ export class TaskManagerRunner implements TaskRunner {
   public async markTaskAsRunning(): Promise<boolean> {
     performance.mark('markTaskAsRunning_start');
 
+    const apmTrans = apm.startTransaction(
+      `taskManager markTaskAsRunning ${this.instance.taskType}`,
+      'taskManager'
+    );
+
     const VERSION_CONFLICT_STATUS = 409;
     const now = new Date();
 
@@ -227,10 +239,12 @@ export class TaskManagerRunner implements TaskRunner {
         );
       }
 
+      if (apmTrans) apmTrans.end('success');
       performanceStopMarkingTaskAsRunning();
       this.onTaskEvent(asTaskMarkRunningEvent(this.id, asOk(this.instance)));
       return true;
     } catch (error) {
+      if (apmTrans) apmTrans.end('failure');
       performanceStopMarkingTaskAsRunning();
       this.onTaskEvent(asTaskMarkRunningEvent(this.id, asErr(error)));
       if (error.statusCode !== VERSION_CONFLICT_STATUS) {
diff --git a/x-pack/plugins/task_manager/server/task_store.ts b/x-pack/plugins/task_manager/server/task_store.ts
index 4f2e97704941f3..3915eeffc55197 100644
--- a/x-pack/plugins/task_manager/server/task_store.ts
+++ b/x-pack/plugins/task_manager/server/task_store.ts
@@ -7,6 +7,7 @@
 /*
  * This module contains helpers for managing the task manager storage layer.
  */
+import apm from 'elastic-apm-node';
 import { Subject, Observable } from 'rxjs';
 import { omit, difference } from 'lodash';
 
@@ -252,6 +253,7 @@ export class TaskStore {
       )
     );
 
+    const apmTrans = apm.startTransaction(`taskManager markAvailableTasksAsClaimed`, 'taskManager');
     const { updated } = await this.updateByQuery(
       asUpdateByQuery({
         query: matchesClauses(
@@ -279,6 +281,7 @@ export class TaskStore {
       }
     );
 
+    if (apmTrans) apmTrans.end();
     return updated;
   }