From 831ae8469453cd2a7c1abdc487ccecb941747ddf Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 25 Jun 2019 16:21:17 -0400
Subject: [PATCH 01/48] Allow mtask definitions to overwrite default setting
 maxAttemps

---
 x-pack/legacy/plugins/task_manager/README.md  |   4 +
 x-pack/legacy/plugins/task_manager/task.ts    |   6 ++
 .../plugins/task_manager/task_manager.ts      |   3 +-
 .../plugins/task_manager/task_runner.test.ts  |  13 ++-
 .../plugins/task_manager/task_runner.ts       |   3 +-
 .../plugins/task_manager/task_store.test.ts   | 101 +++++++++++++++---
 .../legacy/plugins/task_manager/task_store.ts |  58 ++++++----
 7 files changed, 152 insertions(+), 36 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md
index 54992424f9fef9..a160bd0c0a1c1c 100644
--- a/x-pack/legacy/plugins/task_manager/README.md
+++ b/x-pack/legacy/plugins/task_manager/README.md
@@ -69,6 +69,10 @@ taskManager.registerTaskDefinitions({
     // This defaults to 5 minutes.
     timeout: '5m',
 
+    // Optional, how many times to retry if task fails to run.
+    // This defaults to what is configured at the task manager level.
+    maxAttempts: 5,
+
     // The clusterMonitoring task occupies 2 workers, so if the system has 10 worker slots,
     // 5 clusterMonitoring tasks could run concurrently per Kibana instance. This value is
     // overridden by the `override_num_workers` config value, if specified.
diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 9b7191491a27e3..133f2f73852202 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -101,6 +101,11 @@ export interface TaskDefinition {
    */
   timeout?: string;
 
+  /**
+   * Up to how many times the task should retry when it fails to run.
+   */
+  maxAttempts?: number;
+
   /**
    * The numer of workers / slots a running instance of this task occupies.
    * This defaults to 1.
@@ -126,6 +131,7 @@ export const validateTaskDefinition = Joi.object({
   title: Joi.string().optional(),
   description: Joi.string().optional(),
   timeout: Joi.string().default('5m'),
+  maxAttempts: Joi.number().optional(),
   numWorkers: Joi.number()
     .min(1)
     .default(1),
diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts
index 9263747f996436..1ea45b434d75b0 100644
--- a/x-pack/legacy/plugins/task_manager/task_manager.ts
+++ b/x-pack/legacy/plugins/task_manager/task_manager.ts
@@ -59,7 +59,7 @@ export class TaskManager {
       callCluster: server.plugins.elasticsearch.getCluster('admin').callWithInternalUser,
       index: config.get('xpack.task_manager.index'),
       maxAttempts: config.get('xpack.task_manager.max_attempts'),
-      supportedTypes: Object.keys(this.definitions),
+      definitions: this.definitions,
       logger,
       getKibanaUuid: () => config.get('server.uuid'),
     });
@@ -90,7 +90,6 @@ export class TaskManager {
     this.poller = poller;
 
     kbnServer.afterPluginsInit(async () => {
-      store.addSupportedTypes(Object.keys(this.definitions));
       const startPoller = () => {
         return poller
           .start()
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 0909be3b5c4715..08a201744c18b2 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -53,7 +53,7 @@ describe('TaskManagerRunner', () => {
         state: { hey: 'there' },
       },
       definitions: {
-        testtype: {
+        bar: {
           createTaskRunner: () => ({
             async run() {
               throw new Error('Dangit!');
@@ -80,6 +80,15 @@ describe('TaskManagerRunner', () => {
       instance: {
         interval: '10m',
       },
+      definitions: {
+        bar: {
+          createTaskRunner: () => ({
+            async run() {
+              return;
+            },
+          }),
+        },
+      }
     });
 
     await runner.run();
@@ -187,7 +196,7 @@ describe('TaskManagerRunner', () => {
   test('warns if cancel is called on a non-cancellable task', async () => {
     const { runner, logger } = testOpts({
       definitions: {
-        testType: {
+        bar: {
           createTaskRunner: () => ({
             run: async () => undefined,
           }),
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index d5e0196d80864b..ed410c7cd75062 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -212,7 +212,8 @@ export class TaskManagerRunner implements TaskRunner {
   private async processResultForRecurringTask(result: RunResult): Promise<RunResult> {
     // recurring task: update the task instance
     const state = result.state || this.instance.state || {};
-    const status = this.instance.attempts < this.store.maxAttempts ? 'idle' : 'failed';
+    const maxAttempts = this.definition.maxAttempts || this.store.maxAttempts;
+    const status = this.instance.attempts < maxAttempts ? 'idle' : 'failed';
 
     let runAt;
     if (status === 'failed') {
diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts
index 91ed8bcad8a6a9..a12d6e6476a1a9 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts
@@ -10,12 +10,33 @@ import {
   TASK_MANAGER_API_VERSION as API_VERSION,
   TASK_MANAGER_TEMPLATE_VERSION as TEMPLATE_VERSION,
 } from './constants';
-import { TaskInstance, TaskStatus } from './task';
+import { TaskDictionary, SanitizedTaskDefinition, TaskInstance, TaskStatus } from './task';
 import { FetchOpts, TaskStore } from './task_store';
 import { mockLogger } from './test_utils';
 
 const getKibanaUuid = sinon.stub().returns('kibana-uuid-123-test');
 
+const taskDefinitions: TaskDictionary<SanitizedTaskDefinition> = {
+  report: {
+    type: 'report',
+    title: '',
+    numWorkers: 1,
+    createTaskRunner: jest.fn(),
+  },
+  dernstraight: {
+    type: 'dernstraight',
+    title: '',
+    numWorkers: 1,
+    createTaskRunner: jest.fn(),
+  },
+  yawn: {
+    type: 'yawn',
+    title: '',
+    numWorkers: 1,
+    createTaskRunner: jest.fn(),
+  },
+};
+
 describe('TaskStore', () => {
   describe('init', () => {
     test('creates the task manager index', async () => {
@@ -27,7 +48,7 @@ describe('TaskStore', () => {
         logger: mockLogger(),
         index: 'tasky',
         maxAttempts: 2,
-        supportedTypes: ['a', 'b', 'c'],
+        definitions: taskDefinitions,
       });
 
       await store.init();
@@ -65,7 +86,7 @@ describe('TaskStore', () => {
         logger,
         index: 'tasky',
         maxAttempts: 2,
-        supportedTypes: ['a', 'b', 'c'],
+        definitions: taskDefinitions,
       });
 
       await store.init();
@@ -96,7 +117,7 @@ describe('TaskStore', () => {
         logger: mockLogger(),
         index: 'tasky',
         maxAttempts: 2,
-        supportedTypes: ['report', 'dernstraight', 'yawn'],
+        definitions: taskDefinitions,
       });
       await store.init();
       const result = await store.schedule(task);
@@ -171,7 +192,7 @@ describe('TaskStore', () => {
         logger: mockLogger(),
         index: 'tasky',
         maxAttempts: 2,
-        supportedTypes: ['a', 'b', 'c'],
+        definitions: taskDefinitions,
       });
 
       const result = await store.fetch(opts);
@@ -335,7 +356,7 @@ describe('TaskStore', () => {
       const store = new TaskStore({
         callCluster,
         logger: mockLogger(),
-        supportedTypes: ['a', 'b', 'c'],
+        definitions: taskDefinitions,
         index: 'tasky',
         maxAttempts: 2,
         ...opts,
@@ -358,7 +379,7 @@ describe('TaskStore', () => {
         callCluster,
         getKibanaUuid,
         logger: mockLogger(),
-        supportedTypes: ['a', 'b', 'c'],
+        definitions: taskDefinitions,
         index: 'tasky',
         maxAttempts: 2,
       });
@@ -373,12 +394,27 @@ describe('TaskStore', () => {
 
     test('it filters tasks by supported types, maxAttempts, and runAt', async () => {
       const maxAttempts = _.random(2, 43);
+      const customMaxAttempts = _.random(44, 100);
       const index = `index_${_.random(1, 234)}`;
       const { args } = await testFetchAvailableTasks({
         opts: {
           index,
           maxAttempts,
-          supportedTypes: ['foo', 'bar'],
+          definitions: {
+            foo: {
+              type: 'foo',
+              title: '',
+              numWorkers: 1,
+              createTaskRunner: jest.fn(),
+            },
+            bar: {
+              type: 'bar',
+              title: '',
+              numWorkers: 1,
+              maxAttempts: customMaxAttempts,
+              createTaskRunner: jest.fn(),
+            },
+          },
         },
       });
       expect(args).toMatchObject({
@@ -390,10 +426,51 @@ describe('TaskStore', () => {
                 {
                   bool: {
                     must: [
-                      { terms: { 'task.taskType': ['foo', 'bar'] } },
-                      { range: { 'task.attempts': { lte: maxAttempts } } },
+                      {
+                        bool: {
+                          should: [
+                            {
+                              bool: {
+                                must: [
+                                  {
+                                    term: {
+                                      'task.taskType': 'foo',
+                                    },
+                                  },
+                                  {
+                                    range: {
+                                      'task.attempts': {
+                                        lte: maxAttempts,
+                                      },
+                                    },
+                                  },
+                                ],
+                              },
+                            },
+                            {
+                              bool: {
+                                must: [
+                                  {
+                                    term: {
+                                      'task.taskType': 'bar',
+                                    },
+                                  },
+                                  {
+                                    range: {
+                                      'task.attempts': {
+                                        lte: customMaxAttempts,
+                                      },
+                                    },
+                                  },
+                                ],
+                              },
+                            },
+                          ],
+                        },
+                      },
                       { range: { 'task.runAt': { lte: 'now' } } },
                       { range: { 'kibana.apiVersion': { lte: 1 } } },
+                      { term: { 'task.status': 'idle' } },
                     ],
                   },
                 },
@@ -510,7 +587,7 @@ describe('TaskStore', () => {
         logger: mockLogger(),
         index: 'tasky',
         maxAttempts: 2,
-        supportedTypes: ['a', 'b', 'c'],
+        definitions: taskDefinitions,
       });
 
       const result = await store.update(task);
@@ -561,7 +638,7 @@ describe('TaskStore', () => {
         logger: mockLogger(),
         index: 'myindex',
         maxAttempts: 2,
-        supportedTypes: ['a'],
+        definitions: taskDefinitions,
       });
       const result = await store.remove(id);
 
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index d650f3125dc1b7..16b08232c2259c 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -13,14 +13,21 @@ import {
   TASK_MANAGER_TEMPLATE_VERSION as TEMPLATE_VERSION,
 } from './constants';
 import { Logger } from './lib/logger';
-import { ConcreteTaskInstance, ElasticJs, TaskInstance, TaskStatus } from './task';
+import {
+  ConcreteTaskInstance,
+  ElasticJs,
+  SanitizedTaskDefinition,
+  TaskDictionary,
+  TaskInstance,
+  TaskStatus,
+} from './task';
 
 export interface StoreOpts {
   callCluster: ElasticJs;
   getKibanaUuid: () => string;
   index: string;
   maxAttempts: number;
-  supportedTypes: string[];
+  definitions: TaskDictionary<SanitizedTaskDefinition>;
   logger: Logger;
 }
 
@@ -80,7 +87,7 @@ export class TaskStore {
   public getKibanaUuid: () => string;
   public readonly index: string;
   private callCluster: ElasticJs;
-  private supportedTypes: string[];
+  private definitions: TaskDictionary<SanitizedTaskDefinition>;
   private _isInitialized = false; // eslint-disable-line @typescript-eslint/camelcase
   private logger: Logger;
 
@@ -90,28 +97,20 @@ export class TaskStore {
    * @prop {CallCluster} callCluster - The elastic search connection
    * @prop {string} index - The name of the task manager index
    * @prop {number} maxAttempts - The maximum number of attempts before a task will be abandoned
-   * @prop {string[]} supportedTypes - The task types supported by this store
+   * @prop {TaskDefinition} definition - The definition of the task being run
    * @prop {Logger} logger - The task manager logger.
    */
   constructor(opts: StoreOpts) {
     this.callCluster = opts.callCluster;
     this.index = opts.index;
     this.maxAttempts = opts.maxAttempts;
-    this.supportedTypes = opts.supportedTypes;
+    this.definitions = opts.definitions;
     this.logger = opts.logger;
     this.getKibanaUuid = opts.getKibanaUuid;
 
     this.fetchAvailableTasks = this.fetchAvailableTasks.bind(this);
   }
 
-  public addSupportedTypes(types: string[]) {
-    if (!this._isInitialized) {
-      this.supportedTypes = this.supportedTypes.concat(types);
-    } else {
-      throw new Error('Cannot add task types after initialization');
-    }
-  }
-
   /**
    * Initializes the store, ensuring the task manager index template is created
    * and the version is up to date.
@@ -229,11 +228,11 @@ export class TaskStore {
       await this.init();
     }
 
-    if (!this.supportedTypes.includes(taskInstance.taskType)) {
+    if (!this.definitions[taskInstance.taskType]) {
       throw new Error(
-        `Unsupported task type "${
-          taskInstance.taskType
-        }". Supported types are ${this.supportedTypes.join(', ')}`
+        `Unsupported task type "${taskInstance.taskType}". Supported types are ${Object.keys(
+          this.definitions
+        ).join(', ')}`
       );
     }
 
@@ -289,10 +288,31 @@ export class TaskStore {
       query: {
         bool: {
           must: [
-            { terms: { 'task.taskType': this.supportedTypes } },
-            { range: { 'task.attempts': { lte: this.maxAttempts } } },
+            {
+              bool: {
+                should: Object.entries(this.definitions).map(([type, definition]) => ({
+                  bool: {
+                    must: [
+                      {
+                        term: {
+                          'task.taskType': type,
+                        },
+                      },
+                      {
+                        range: {
+                          'task.attempts': {
+                            lte: definition.maxAttempts || this.maxAttempts,
+                          },
+                        },
+                      },
+                    ],
+                  },
+                })),
+              },
+            },
             { range: { 'task.runAt': { lte: 'now' } } },
             { range: { 'kibana.apiVersion': { lte: API_VERSION } } },
+            { term: { 'task.status': 'idle' } },
           ],
         },
       },

From 05a2061a10a6a82d6c0841cc8c7d9e724bc4078b Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 25 Jun 2019 16:42:38 -0400
Subject: [PATCH 02/48] Leverage scheduledAt from task manager

---
 .../lib/get_create_task_runner_function.test.ts      | 10 ++++++++--
 .../server/lib/get_create_task_runner_function.ts    | 12 +++++-------
 2 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
index 5a7cebddfcdf1f..b6e4ba510d51fa 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
@@ -6,6 +6,7 @@
 
 import Joi from 'joi';
 import { AlertExecuteOptions } from '../types';
+import { ConcreteTaskInstance } from '../../../task_manager';
 import { SavedObjectsClientMock } from '../../../../../../src/core/server/mocks';
 import { getCreateTaskRunnerFunction } from './get_create_task_runner_function';
 
@@ -36,8 +37,14 @@ const getCreateTaskRunnerFunctionParams = {
   internalSavedObjectsRepository: savedObjectsClient,
 };
 
-const mockedTaskInstance = {
+const mockedTaskInstance: ConcreteTaskInstance = {
+  id: '',
+  attempts: 0,
+  status: 'running',
+  primaryTerm: 0,
+  sequenceNumber: 1,
   runAt: mockedLastRunAt,
+  scheduledAt: mockedLastRunAt,
   state: {
     scheduledRunAt: mockedLastRunAt,
   },
@@ -89,7 +96,6 @@ Object {
     "alertInstances": Object {},
     "alertTypeState": undefined,
     "previousScheduledRunAt": 2019-06-03T18:55:20.982Z,
-    "scheduledRunAt": 2019-06-03T18:55:30.982Z,
   },
 }
 `);
diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
index 311b8b5dd05642..364cc1da309f69 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
@@ -7,7 +7,7 @@
 import { SavedObjectsClientContract } from 'src/core/server';
 import { ActionsPlugin } from '../../../actions';
 import { AlertType, Services, AlertServices } from '../types';
-import { TaskInstance } from '../../../task_manager';
+import { ConcreteTaskInstance } from '../../../task_manager';
 import { createFireHandler } from './create_fire_handler';
 import { createAlertInstanceFactory } from './create_alert_instance_factory';
 import { AlertInstance } from './alert_instance';
@@ -22,7 +22,7 @@ interface CreateTaskRunnerFunctionOptions {
 }
 
 interface TaskRunnerOptions {
-  taskInstance: TaskInstance;
+  taskInstance: ConcreteTaskInstance;
 }
 
 export function getCreateTaskRunnerFunction({
@@ -66,7 +66,7 @@ export function getCreateTaskRunnerFunction({
           services: alertTypeServices,
           params: validatedAlertTypeParams,
           state: taskInstance.state.alertTypeState || {},
-          scheduledRunAt: taskInstance.state.scheduledRunAt,
+          scheduledRunAt: taskInstance.scheduledAt,
           previousScheduledRunAt: taskInstance.state.previousScheduledRunAt,
         });
 
@@ -88,7 +88,7 @@ export function getCreateTaskRunnerFunction({
         );
 
         const nextRunAt = getNextRunAt(
-          new Date(taskInstance.state.scheduledRunAt),
+          new Date(taskInstance.scheduledAt),
           alertSavedObject.attributes.interval
         );
 
@@ -96,9 +96,7 @@ export function getCreateTaskRunnerFunction({
           state: {
             alertTypeState,
             alertInstances,
-            // We store nextRunAt ourselves since task manager changes runAt when executing a task
-            scheduledRunAt: nextRunAt,
-            previousScheduledRunAt: taskInstance.state.scheduledRunAt,
+            previousScheduledRunAt: taskInstance.scheduledAt,
           },
           runAt: nextRunAt,
         };

From d102bececc6461bdba34f1b46e68c1b77d1c7266 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 26 Jun 2019 08:38:21 -0400
Subject: [PATCH 03/48] Treat maxAttempts like attempts and not retries

---
 x-pack/legacy/plugins/task_manager/task_runner.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index ed410c7cd75062..ca7059e5b87198 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -211,9 +211,10 @@ export class TaskManagerRunner implements TaskRunner {
 
   private async processResultForRecurringTask(result: RunResult): Promise<RunResult> {
     // recurring task: update the task instance
+    const attempts = result.error ? this.instance.attempts + 1 : 0;
     const state = result.state || this.instance.state || {};
     const maxAttempts = this.definition.maxAttempts || this.store.maxAttempts;
-    const status = this.instance.attempts < maxAttempts ? 'idle' : 'failed';
+    const status = attempts < maxAttempts ? 'idle' : 'failed';
 
     let runAt;
     if (status === 'failed') {
@@ -232,7 +233,7 @@ export class TaskManagerRunner implements TaskRunner {
       runAt,
       state,
       status,
-      attempts: result.error ? this.instance.attempts + 1 : 0,
+      attempts,
     });
 
     return result;

From 5dc80d77330bfc4d66c4f6b857adf8dbc4d8bbfe Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 26 Jun 2019 08:53:56 -0400
Subject: [PATCH 04/48] Add support for second intervals

---
 x-pack/legacy/plugins/task_manager/README.md  |  2 +-
 .../task_manager/lib/intervals.test.ts        | 26 ++++++++++++--
 .../plugins/task_manager/lib/intervals.ts     | 35 +++++++++++++++++--
 3 files changed, 56 insertions(+), 7 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md
index a160bd0c0a1c1c..1cf30537ec7fc1 100644
--- a/x-pack/legacy/plugins/task_manager/README.md
+++ b/x-pack/legacy/plugins/task_manager/README.md
@@ -165,7 +165,7 @@ The data stored for a task instance looks something like this:
   runAt: "2020-07-24T17:34:35.272Z",
 
   // Indicates that this is a recurring task. We currently only support
-  // 1 minute granularity.
+  // minute syntax `5m` or second syntax `10s`.
   interval: '5m',
 
   // How many times this task has been unsuccesfully attempted,
diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
index bba5cdf8591ff8..4552aa9d04b72a 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
@@ -5,7 +5,7 @@
  */
 
 import _ from 'lodash';
-import { assertValidInterval, intervalFromNow, minutesFromNow } from './intervals';
+import { assertValidInterval, intervalFromNow, minutesFromNow, secondsFromNow } from './intervals';
 
 describe('taskIntervals', () => {
   describe('assertValidInterval', () => {
@@ -13,7 +13,11 @@ describe('taskIntervals', () => {
       expect(() => assertValidInterval(`${_.random(1000)}m`)).not.toThrow();
     });
 
-    test('it rejects intervals are not of the form `Nm`', () => {
+    test('it accepts intervals in the form `Ns`', () => {
+      expect(() => assertValidInterval(`${_.random(1000)}s`)).not.toThrow();
+    });
+
+    test('it rejects intervals are not of the form `Nm` or `Ns`', () => {
       expect(() => assertValidInterval(`5m 2s`)).toThrow(
         /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/
       );
@@ -31,7 +35,14 @@ describe('taskIntervals', () => {
       expect(Math.abs(nextRun - expected)).toBeLessThan(100);
     });
 
-    test('it rejects intervals are not of the form `Nm`', () => {
+    test('it returns the current date plus n seconds', () => {
+      const secs = _.random(1, 100);
+      const expected = Date.now() + secs * 1000;
+      const nextRun = intervalFromNow(`${secs}s`)!.getTime();
+      expect(Math.abs(nextRun - expected)).toBeLessThan(100);
+    });
+
+    test('it rejects intervals are not of the form `Nm` or `Ns`', () => {
       expect(() => intervalFromNow(`5m 2s`)).toThrow(
         /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/
       );
@@ -49,4 +60,13 @@ describe('taskIntervals', () => {
       expect(Math.abs(nextRun - expected)).toBeLessThan(100);
     });
   });
+
+  describe('secondsFromNow', () => {
+    test('it returns the current date plus a number of seconds', () => {
+      const secs = _.random(1, 100);
+      const expected = Date.now() + secs * 1000;
+      const nextRun = secondsFromNow(secs).getTime();
+      expect(Math.abs(nextRun - expected)).toBeLessThan(100);
+    });
+  });
 });
diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
index f095c336098f95..16e548e4dd653a 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
@@ -6,7 +6,7 @@
 
 /**
  * Returns a date that is the specified interval from now. Currently,
- * only minute-intervals are supported.
+ * only minute-intervals and second-intervals are supported.
  *
  * @param {string} interval - An interval of the form `Nm` such as `5m`
  */
@@ -17,6 +17,10 @@ export function intervalFromNow(interval?: string): Date | undefined {
 
   assertValidInterval(interval);
 
+  if (isSeconds(interval)) {
+    return secondsFromNow(parseInterval(interval));
+  }
+
   return minutesFromNow(parseInterval(interval));
 }
 
@@ -33,13 +37,30 @@ export function minutesFromNow(mins: number): Date {
   return now;
 }
 
+/**
+ * Returns a date that is secs seconds from now.
+ *
+ * @param secs The number of seconds from now
+ */
+export function secondsFromNow(secs: number): Date {
+  const now = new Date();
+
+  now.setSeconds(now.getSeconds() + secs);
+
+  return now;
+}
+
 /**
  * Verifies that the specified interval matches our expected format.
  *
- * @param {string} interval - An interval such as `5m`
+ * @param {string} interval - An interval such as `5m` or `10s`
  */
 export function assertValidInterval(interval: string) {
-  if (/^[0-9]+m$/.test(interval)) {
+  if (isMinutes(interval)) {
+    return interval;
+  }
+
+  if (isSeconds(interval)) {
     return interval;
   }
 
@@ -51,3 +72,11 @@ export function assertValidInterval(interval: string) {
 function parseInterval(interval: string) {
   return parseInt(interval, 10);
 }
+
+function isMinutes(interval: string) {
+  return /^[0-9]+m$/.test(interval);
+}
+
+function isSeconds(interval: string) {
+  return /^[0-9]+s$/.test(interval);
+}

From dbf5159064b35b809835bf7e43a7c95c2e45b08b Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 26 Jun 2019 12:15:05 -0400
Subject: [PATCH 05/48] Min 1 attempt

---
 x-pack/legacy/plugins/task_manager/README.md | 4 ++--
 x-pack/legacy/plugins/task_manager/index.js  | 2 +-
 x-pack/legacy/plugins/task_manager/task.ts   | 4 +++-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md
index 1cf30537ec7fc1..df5c03cc1b2f20 100644
--- a/x-pack/legacy/plugins/task_manager/README.md
+++ b/x-pack/legacy/plugins/task_manager/README.md
@@ -38,7 +38,7 @@ If a task specifies a higher `numWorkers` than the system supports, the system's
 
 The task_manager can be configured via `taskManager` config options (e.g. `taskManager.maxAttempts`):
 
-- `max_attempts` - How many times a failing task instance will be retried before it is never run again
+- `max_attempts` - The maximum number of times a task will be attempted before being abandoned as failed
 - `poll_interval` - How often the background worker should check the task_manager index for more work
 - `index` - The name of the index that the task_manager
 - `max_workers` - The maximum number of tasks a Kibana will run concurrently (defaults to 10)
@@ -69,7 +69,7 @@ taskManager.registerTaskDefinitions({
     // This defaults to 5 minutes.
     timeout: '5m',
 
-    // Optional, how many times to retry if task fails to run.
+    // Optional, how many attempts before marking task as failed.
     // This defaults to what is configured at the task manager level.
     maxAttempts: 5,
 
diff --git a/x-pack/legacy/plugins/task_manager/index.js b/x-pack/legacy/plugins/task_manager/index.js
index e0116820a3e050..a6f2b8adee81b5 100644
--- a/x-pack/legacy/plugins/task_manager/index.js
+++ b/x-pack/legacy/plugins/task_manager/index.js
@@ -16,7 +16,7 @@ export function taskManager(kibana) {
         enabled: Joi.boolean().default(true),
         max_attempts: Joi.number()
           .description('The maximum number of times a task will be attempted before being abandoned as failed')
-          .min(0) // no retries
+          .min(1)
           .default(3),
         poll_interval: Joi.number()
           .description('How often, in milliseconds, the task manager will look for more work.')
diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 133f2f73852202..7163339ffc2eec 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -131,7 +131,9 @@ export const validateTaskDefinition = Joi.object({
   title: Joi.string().optional(),
   description: Joi.string().optional(),
   timeout: Joi.string().default('5m'),
-  maxAttempts: Joi.number().optional(),
+  maxAttempts: Joi.number()
+    .min(1)
+    .optional(),
   numWorkers: Joi.number()
     .min(1)
     .default(1),

From 557bff80c9a28317971c1e7da465bf9d67b2e9ad Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 27 Jun 2019 07:53:06 -0400
Subject: [PATCH 06/48] Reverse relying on scheduledAt

---
 .../server/lib/get_create_task_runner_function.test.ts    | 1 +
 .../server/lib/get_create_task_runner_function.ts         | 8 +++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
index b6e4ba510d51fa..0386c0401f3907 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
@@ -96,6 +96,7 @@ Object {
     "alertInstances": Object {},
     "alertTypeState": undefined,
     "previousScheduledRunAt": 2019-06-03T18:55:20.982Z,
+    "scheduledRunAt": 2019-06-03T18:55:30.982Z,
   },
 }
 `);
diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
index 364cc1da309f69..c4f4545a68d05c 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
@@ -66,7 +66,7 @@ export function getCreateTaskRunnerFunction({
           services: alertTypeServices,
           params: validatedAlertTypeParams,
           state: taskInstance.state.alertTypeState || {},
-          scheduledRunAt: taskInstance.scheduledAt,
+          scheduledRunAt: taskInstance.state.scheduledRunAt,
           previousScheduledRunAt: taskInstance.state.previousScheduledRunAt,
         });
 
@@ -88,7 +88,7 @@ export function getCreateTaskRunnerFunction({
         );
 
         const nextRunAt = getNextRunAt(
-          new Date(taskInstance.scheduledAt),
+          new Date(taskInstance.state.scheduledRunAt),
           alertSavedObject.attributes.interval
         );
 
@@ -96,7 +96,9 @@ export function getCreateTaskRunnerFunction({
           state: {
             alertTypeState,
             alertInstances,
-            previousScheduledRunAt: taskInstance.scheduledAt,
+            // We store nextRunAt ourselves since task manager changes runAt when executing a task
+            scheduledRunAt: nextRunAt,
+            previousScheduledRunAt: taskInstance.state.scheduledRunAt,
           },
           runAt: nextRunAt,
         };

From 418b4e1de20c74eca82635829fff8a62e87eccb0 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 27 Jun 2019 08:40:29 -0400
Subject: [PATCH 07/48] Add new startedAt attribute in task manager that keeps
 track when task started running

---
 x-pack/legacy/plugins/alerting/README.md      | 12 +++++-----
 .../alerting/server/alerts_client.test.ts     |  5 ++--
 .../plugins/alerting/server/alerts_client.ts  |  4 ++--
 .../get_create_task_runner_function.test.ts   |  8 +++----
 .../lib/get_create_task_runner_function.ts    | 10 ++++----
 .../legacy/plugins/alerting/server/types.ts   |  4 ++--
 .../task_manager/lib/middleware.test.ts       |  3 +++
 x-pack/legacy/plugins/task_manager/task.ts    | 14 +++++++++++
 .../plugins/task_manager/task_poller.test.ts  | 24 ++++++++++++++++++-
 .../plugins/task_manager/task_runner.test.ts  |  1 +
 .../plugins/task_manager/task_runner.ts       |  2 ++
 .../plugins/task_manager/task_store.test.ts   |  1 +
 .../legacy/plugins/task_manager/task_store.ts |  4 ++++
 13 files changed, 69 insertions(+), 23 deletions(-)

diff --git a/x-pack/legacy/plugins/alerting/README.md b/x-pack/legacy/plugins/alerting/README.md
index 27bf614da70447..8c8351cec2141f 100644
--- a/x-pack/legacy/plugins/alerting/README.md
+++ b/x-pack/legacy/plugins/alerting/README.md
@@ -52,8 +52,8 @@ This is the primary function for an alert type. Whenever the alert needs to exec
 |services.callCluster(path, opts)|Use this to do Elasticsearch queries on the cluster Kibana connects to. This function is the same as any other `callCluster` in Kibana.<br><br>**NOTE**: This currently authenticates as the Kibana internal user, but will change in a future PR.|
 |services.savedObjectsClient|This is an instance of the saved objects client. This provides the ability to do CRUD on any saved objects within the same space the alert lives in.<br><br>**NOTE**: This currently only works when security is disabled. A future PR will add support for enabled security using Elasticsearch API tokens.|
 |services.log(tags, [data], [timestamp])|Use this to create server logs. (This is the same function as server.log)|
-|scheduledRunAt|The date and time the alert type execution was scheduled to be called.|
-|previousScheduledRunAt|The previous date and time the alert type was scheduled to be called.|
+|startedAt|The date and time the alert type started execution.|
+|previousStartedAt|The previous date and time the alert type started execution.|
 |params|Parameters for the execution. This is where the parameters you require will be passed in. (example threshold). Use alert type validation to ensure values are set before execution.|
 |state|State returned from previous execution. This is the alert level state. What the executor returns will be serialized and provided here at the next execution.|
 
@@ -74,8 +74,8 @@ server.plugins.alerting.registerType({
 			.required(),
 	},
 	async execute({
-		scheduledRunAt,
-		previousScheduledRunAt,
+		startedAt,
+		previousStartedAt,
 		services,
 		params,
 		state,
@@ -133,8 +133,8 @@ server.plugins.alerting.registerType({
 			.required(),
 	},
 	async execute({
-		scheduledRunAt,
-		previousScheduledRunAt,
+		startedAt,
+		previousStartedAt,
 		services,
 		params,
 		state,
diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
index db7e70948d1de9..147fcb5c88c86f 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
@@ -98,6 +98,7 @@ describe('create()', () => {
       attempts: 1,
       status: 'idle',
       runAt: new Date(),
+      startedAt: null,
       state: {},
       params: {},
     });
@@ -182,8 +183,8 @@ Array [
     "state": Object {
       "alertInstances": Object {},
       "alertTypeState": Object {},
-      "previousScheduledRunAt": null,
-      "scheduledRunAt": 2019-02-12T21:01:22.479Z,
+      "previousStartedAt": null,
+      "startedAt": 2019-02-12T21:01:22.479Z,
     },
     "taskType": "alerting:123",
   },
diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.ts
index d1b2c5b9b8ef80..308f9930bee955 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.ts
@@ -171,8 +171,8 @@ export class AlertsClient {
       state: {
         // This is here because we can't rely on the task manager's internal runAt.
         // It changes it for timeout, etc when a task is running.
-        scheduledRunAt: new Date(Date.now() + alert.interval),
-        previousScheduledRunAt: null,
+        startedAt: new Date(Date.now() + alert.interval),
+        previousStartedAt: null,
         alertTypeState: {},
         alertInstances: {},
       },
diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
index 0386c0401f3907..c4e46de4082b17 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
@@ -45,8 +45,9 @@ const mockedTaskInstance: ConcreteTaskInstance = {
   sequenceNumber: 1,
   runAt: mockedLastRunAt,
   scheduledAt: mockedLastRunAt,
+  startedAt: mockedLastRunAt,
   state: {
-    scheduledRunAt: mockedLastRunAt,
+    startedAt: mockedLastRunAt,
   },
   taskType: 'alerting:test',
   params: {
@@ -95,8 +96,7 @@ Object {
   "state": Object {
     "alertInstances": Object {},
     "alertTypeState": undefined,
-    "previousScheduledRunAt": 2019-06-03T18:55:20.982Z,
-    "scheduledRunAt": 2019-06-03T18:55:30.982Z,
+    "previousStartedAt": 2019-06-03T18:55:20.982Z,
   },
 }
 `);
@@ -107,7 +107,7 @@ Object {
   "bar": true,
 }
 `);
-  expect(call.scheduledRunAt).toMatchInlineSnapshot(`2019-06-03T18:55:20.982Z`);
+  expect(call.startedAt).toMatchInlineSnapshot(`2019-06-03T18:55:20.982Z`);
   expect(call.state).toMatchInlineSnapshot(`Object {}`);
   expect(call.services.alertInstanceFactory).toBeTruthy();
   expect(call.services.callCluster).toBeTruthy();
diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
index c4f4545a68d05c..a91f9f8cceb744 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
@@ -66,8 +66,8 @@ export function getCreateTaskRunnerFunction({
           services: alertTypeServices,
           params: validatedAlertTypeParams,
           state: taskInstance.state.alertTypeState || {},
-          scheduledRunAt: taskInstance.state.scheduledRunAt,
-          previousScheduledRunAt: taskInstance.state.previousScheduledRunAt,
+          startedAt: taskInstance.startedAt!,
+          previousStartedAt: taskInstance.state.previousStartedAt,
         });
 
         await Promise.all(
@@ -88,7 +88,7 @@ export function getCreateTaskRunnerFunction({
         );
 
         const nextRunAt = getNextRunAt(
-          new Date(taskInstance.state.scheduledRunAt),
+          new Date(taskInstance.startedAt!),
           alertSavedObject.attributes.interval
         );
 
@@ -96,9 +96,7 @@ export function getCreateTaskRunnerFunction({
           state: {
             alertTypeState,
             alertInstances,
-            // We store nextRunAt ourselves since task manager changes runAt when executing a task
-            scheduledRunAt: nextRunAt,
-            previousScheduledRunAt: taskInstance.state.scheduledRunAt,
+            previousStartedAt: taskInstance.startedAt!,
           },
           runAt: nextRunAt,
         };
diff --git a/x-pack/legacy/plugins/alerting/server/types.ts b/x-pack/legacy/plugins/alerting/server/types.ts
index 83724280553eea..2fdcf69bc7c76f 100644
--- a/x-pack/legacy/plugins/alerting/server/types.ts
+++ b/x-pack/legacy/plugins/alerting/server/types.ts
@@ -29,8 +29,8 @@ export interface AlertServices extends Services {
 }
 
 export interface AlertExecuteOptions {
-  scheduledRunAt: Date;
-  previousScheduledRunAt?: Date;
+  startedAt: Date;
+  previousStartedAt?: Date;
   services: AlertServices;
   params: Record<string, any>;
   state: State;
diff --git a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts
index 4add3a81501a1b..ee82a036c7b444 100644
--- a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts
@@ -26,6 +26,7 @@ const getMockConcreteTaskInstance = () => {
     status: TaskStatus;
     runAt: Date;
     scheduledAt: Date;
+    startedAt: Date | null;
     state: any;
     taskType: string;
     params: any;
@@ -37,6 +38,7 @@ const getMockConcreteTaskInstance = () => {
     status: 'idle',
     runAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()),
     scheduledAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()),
+    startedAt: null,
     state: {},
     taskType: 'nice_task',
     params: { abc: 'def' },
@@ -156,6 +158,7 @@ Object {
     "runAt": 2018-09-18T05:33:09.588Z,
     "scheduledAt": 2018-09-18T05:33:09.588Z,
     "sequenceNumber": 1,
+    "startedAt": undefined,
     "state": Object {},
     "status": "idle",
     "taskType": "nice_task",
diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 7163339ffc2eec..67f9792bd50257 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -172,6 +172,13 @@ export interface TaskInstance {
    */
   scheduledAt?: Date;
 
+  /**
+   * The date and time that this task started execution. This is used to determine
+   * the "real" runAt that ended up running the task. This value is only set
+   * when status is set to "running".
+   */
+  startedAt?: Date | null;
+
   /**
    * The date and time that this task is scheduled to be run. It is not
    * guaranteed to run at this time, but it is guaranteed not to run earlier
@@ -252,6 +259,13 @@ export interface ConcreteTaskInstance extends TaskInstance {
    */
   runAt: Date;
 
+  /**
+   * The date and time that this task started execution. This is used to determine
+   * the "real" runAt that ended up running the task. This value is only set
+   * when status is set to "running".
+   */
+  startedAt: Date | null;
+
   /**
    * The state passed into the task's run function, and returned by the previous
    * run. If there was no previous run, or if the previous run did not return
diff --git a/x-pack/legacy/plugins/task_manager/task_poller.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.test.ts
index 23f604e861c147..f61d569cead4b7 100644
--- a/x-pack/legacy/plugins/task_manager/task_poller.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_poller.test.ts
@@ -9,9 +9,31 @@ import sinon from 'sinon';
 import { TaskPoller } from './task_poller';
 import { TaskStore } from './task_store';
 import { mockLogger, resolvable, sleep } from './test_utils';
+import { TaskDictionary, SanitizedTaskDefinition } from './task';
 
 let store: TaskStore;
 
+const taskDefinitions: TaskDictionary<SanitizedTaskDefinition> = {
+  report: {
+    type: 'report',
+    title: '',
+    numWorkers: 1,
+    createTaskRunner: jest.fn(),
+  },
+  dernstraight: {
+    type: 'dernstraight',
+    title: '',
+    numWorkers: 1,
+    createTaskRunner: jest.fn(),
+  },
+  yawn: {
+    type: 'yawn',
+    title: '',
+    numWorkers: 1,
+    createTaskRunner: jest.fn(),
+  },
+};
+
 describe('TaskPoller', () => {
   beforeEach(() => {
     const callCluster = sinon.stub();
@@ -23,7 +45,7 @@ describe('TaskPoller', () => {
       logger: mockLogger(),
       index: 'tasky',
       maxAttempts: 2,
-      supportedTypes: ['a', 'b', 'c'],
+      definitions: taskDefinitions,
     });
   });
 
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 08a201744c18b2..af7349ad7c906f 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -243,6 +243,7 @@ describe('TaskManagerRunner', () => {
           primaryTerm: 32,
           runAt: new Date(),
           scheduledAt: new Date(),
+          startedAt: null,
           attempts: 0,
           params: {},
           scope: ['reporting'],
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index ca7059e5b87198..a0eaff9c9d1276 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -171,6 +171,7 @@ export class TaskManagerRunner implements TaskRunner {
       this.instance = await this.store.update({
         ...this.instance,
         status: 'running',
+        startedAt: new Date(),
         runAt: intervalFromNow(this.definition.timeout)!,
       });
 
@@ -233,6 +234,7 @@ export class TaskManagerRunner implements TaskRunner {
       runAt,
       state,
       status,
+      startedAt: null,
       attempts,
     });
 
diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts
index a12d6e6476a1a9..ef168bd5b3f192 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts
@@ -566,6 +566,7 @@ describe('TaskStore', () => {
       const task = {
         runAt,
         scheduledAt: runAt,
+        startedAt: null,
         id: 'task:324242',
         params: { hello: 'world' },
         state: { foo: 'bar' },
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 16b08232c2259c..29b47dfef426c0 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -67,6 +67,7 @@ export interface RawTaskDoc {
       taskType: string;
       scheduledAt: Date;
       runAt: Date;
+      startedAt: Date | null;
       interval?: string;
       attempts: number;
       status: TaskStatus;
@@ -178,6 +179,7 @@ export class TaskStore {
                 taskType: { type: 'keyword' },
                 scheduledAt: { type: 'date' },
                 runAt: { type: 'date' },
+                startedAt: { type: 'date' },
                 interval: { type: 'text' },
                 attempts: { type: 'integer' },
                 status: { type: 'keyword' },
@@ -253,6 +255,7 @@ export class TaskStore {
       attempts: 0,
       status: task.status,
       scheduledAt: task.scheduledAt,
+      startedAt: null,
       runAt: task.runAt,
       state: taskInstance.state || {},
     };
@@ -425,6 +428,7 @@ function rawSource(doc: TaskInstance, store: TaskStore) {
     state: JSON.stringify(doc.state || {}),
     attempts: (doc as ConcreteTaskInstance).attempts || 0,
     scheduledAt: doc.scheduledAt || new Date(),
+    startedAt: doc.startedAt || null,
     runAt: doc.runAt || new Date(),
     status: (doc as ConcreteTaskInstance).status || 'idle',
   };

From 0da2cf3fb36148e013c698eebf6f316ce0d2338d Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 27 Jun 2019 13:18:29 -0400
Subject: [PATCH 08/48] Don't extend runAt when claiming a task

---
 x-pack/legacy/plugins/task_manager/README.md  |  4 +-
 .../plugins/task_manager/lib/intervals.ts     | 49 +++++++++++++++++++
 x-pack/legacy/plugins/task_manager/task.ts    |  2 +-
 .../plugins/task_manager/task_runner.ts       | 13 +++--
 .../legacy/plugins/task_manager/task_store.ts | 28 +++++++++--
 5 files changed, 82 insertions(+), 14 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md
index df5c03cc1b2f20..0107ac14559a5f 100644
--- a/x-pack/legacy/plugins/task_manager/README.md
+++ b/x-pack/legacy/plugins/task_manager/README.md
@@ -19,7 +19,7 @@ At a high-level, the task manager works like this:
   - `attempts` is less than the configured threshold
 - Attempt to claim the task by using optimistic concurrency to set:
   - status to `running`
-  - `runAt` to now + the timeout specified by the task
+  - `startedAt` to now
 - Execute the task, if the previous claim succeeded
 - If the task fails, increment the `attempts` count and reschedule it
 - If the task succeeds:
@@ -64,7 +64,7 @@ taskManager.registerTaskDefinitions({
     // Optional, human-friendly, more detailed description
     description: 'Amazing!!',
 
-    // Optional, how long, in minutes, the system should wait before
+    // Optional, how long, in minutes or seconds, the system should wait before
     // a running instance of this task is considered to be timed out.
     // This defaults to 5 minutes.
     timeout: '5m',
diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
index 16e548e4dd653a..0d210a413de2d1 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
@@ -24,6 +24,27 @@ export function intervalFromNow(interval?: string): Date | undefined {
   return minutesFromNow(parseInterval(interval));
 }
 
+/**
+ * Returns a date that is the specified interval from given date. Currently,
+ * only minute-intervals and second-intervals are supported.
+ *
+ * @param {Date} date - The date to add interval to
+ * @param {string} interval - An interval of the form `Nm` such as `5m`
+ */
+export function intervalFromDate(date: Date, interval?: string): Date | undefined {
+  if (interval === undefined) {
+    return;
+  }
+
+  assertValidInterval(interval);
+
+  if (isSeconds(interval)) {
+    return secondsFromDate(date, parseInterval(interval));
+  }
+
+  return minutesFromDate(date, parseInterval(interval));
+}
+
 /**
  * Returns a date that is mins minutes from now.
  *
@@ -37,6 +58,20 @@ export function minutesFromNow(mins: number): Date {
   return now;
 }
 
+/**
+ * Returns a date that is mins minutes from given date.
+ *
+ * @param date The date to add minutes to
+ * @param mins The number of mintues from given date
+ */
+export function minutesFromDate(date: Date, mins: number): Date {
+  const result = new Date(date.valueOf());
+
+  result.setMinutes(result.getMinutes() + mins);
+
+  return result;
+}
+
 /**
  * Returns a date that is secs seconds from now.
  *
@@ -50,6 +85,20 @@ export function secondsFromNow(secs: number): Date {
   return now;
 }
 
+/**
+ * Returns a date that is secs seconds from given date.
+ *
+ * @param date The date to add seconds to
+ * @param secs The number of seconds from given date
+ */
+export function secondsFromDate(date: Date, secs: number): Date {
+  const result = new Date(date.valueOf());
+
+  result.setSeconds(result.getSeconds() + secs);
+
+  return result;
+}
+
 /**
  * Verifies that the specified interval matches our expected format.
  *
diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 67f9792bd50257..494bd59e8460ac 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -94,7 +94,7 @@ export interface TaskDefinition {
   description?: string;
 
   /**
-   * How long, in minutes, the system should wait for the task to complete
+   * How long, in minutes or seconds, the system should wait for the task to complete
    * before it is considered to be timed out. (e.g. '5m', the default). If
    * the task takes longer than this, Kibana will send it a kill command and
    * the task will be re-attempted.
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index a0eaff9c9d1276..d3f4132fdabb09 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -11,7 +11,7 @@
  */
 
 import Joi from 'joi';
-import { intervalFromNow, minutesFromNow } from './lib/intervals';
+import { intervalFromNow, minutesFromNow, intervalFromDate } from './lib/intervals';
 import { Logger } from './lib/logger';
 import { BeforeRunFunction } from './lib/middleware';
 import {
@@ -119,7 +119,7 @@ export class TaskManagerRunner implements TaskRunner {
    * Gets whether or not this task has run longer than its expiration setting allows.
    */
   public get isExpired() {
-    return this.instance.runAt < new Date();
+    return intervalFromDate(this.instance.startedAt!, this.definition.timeout)! < new Date();
   }
 
   /**
@@ -172,7 +172,7 @@ export class TaskManagerRunner implements TaskRunner {
         ...this.instance,
         status: 'running',
         startedAt: new Date(),
-        runAt: intervalFromNow(this.definition.timeout)!,
+        attempts: this.instance.attempts + 1,
       });
 
       return true;
@@ -212,10 +212,9 @@ export class TaskManagerRunner implements TaskRunner {
 
   private async processResultForRecurringTask(result: RunResult): Promise<RunResult> {
     // recurring task: update the task instance
-    const attempts = result.error ? this.instance.attempts + 1 : 0;
     const state = result.state || this.instance.state || {};
     const maxAttempts = this.definition.maxAttempts || this.store.maxAttempts;
-    const status = attempts < maxAttempts ? 'idle' : 'failed';
+    const status = this.instance.attempts < maxAttempts ? 'idle' : 'failed';
 
     let runAt;
     if (status === 'failed') {
@@ -226,7 +225,7 @@ export class TaskManagerRunner implements TaskRunner {
         result.runAt ||
         intervalFromNow(this.instance.interval) ||
         // when result.error is truthy, then we're retrying because it failed
-        minutesFromNow((this.instance.attempts + 1) * 5); // incrementally backs off an extra 5m per failure
+        minutesFromNow(this.instance.attempts * 5); // incrementally backs off an extra 5m per failure
     }
 
     await this.store.update({
@@ -235,7 +234,7 @@ export class TaskManagerRunner implements TaskRunner {
       state,
       status,
       startedAt: null,
-      attempts,
+      attempts: result.error ? this.instance.attempts : 0,
     });
 
     return result;
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 29b47dfef426c0..37f3a816bb80e3 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -21,6 +21,7 @@ import {
   TaskInstance,
   TaskStatus,
 } from './task';
+import { intervalFromNow } from './lib/intervals';
 
 export interface StoreOpts {
   callCluster: ElasticJs;
@@ -291,6 +292,8 @@ export class TaskStore {
       query: {
         bool: {
           must: [
+            { range: { 'task.runAt': { lte: 'now' } } },
+            { range: { 'kibana.apiVersion': { lte: API_VERSION } } },
             {
               bool: {
                 should: Object.entries(this.definitions).map(([type, definition]) => ({
@@ -304,7 +307,7 @@ export class TaskStore {
                       {
                         range: {
                           'task.attempts': {
-                            lte: definition.maxAttempts || this.maxAttempts,
+                            lt: definition.maxAttempts || this.maxAttempts,
                           },
                         },
                       },
@@ -313,9 +316,26 @@ export class TaskStore {
                 })),
               },
             },
-            { range: { 'task.runAt': { lte: 'now' } } },
-            { range: { 'kibana.apiVersion': { lte: API_VERSION } } },
-            { term: { 'task.status': 'idle' } },
+            {
+              bool: {
+                should: [
+                  { term: { 'task.status': 'idle' } },
+                  Object.entries(this.definitions).map(([type, definition]) => ({
+                    bool: {
+                      must: [
+                        { term: { 'task.status': 'running' } },
+                        { term: { 'task.taskType': type } },
+                        {
+                          range: {
+                            'task.startedAt': { lte: `now-${definition.timeout}` },
+                          },
+                        },
+                      ],
+                    },
+                  })),
+                ],
+              },
+            },
           ],
         },
       },

From 507f4e8419ecfcca65e127f1ad7e5a8a5030748a Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 27 Jun 2019 14:52:02 -0400
Subject: [PATCH 09/48] Remove startedAt from state

---
 x-pack/legacy/plugins/alerting/server/alerts_client.test.ts | 1 -
 x-pack/legacy/plugins/alerting/server/alerts_client.ts      | 3 ---
 2 files changed, 4 deletions(-)

diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
index 147fcb5c88c86f..9a22542609073c 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
@@ -184,7 +184,6 @@ Array [
       "alertInstances": Object {},
       "alertTypeState": Object {},
       "previousStartedAt": null,
-      "startedAt": 2019-02-12T21:01:22.479Z,
     },
     "taskType": "alerting:123",
   },
diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.ts
index 308f9930bee955..6592f7c3a56722 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.ts
@@ -169,9 +169,6 @@ export class AlertsClient {
         basePath,
       },
       state: {
-        // This is here because we can't rely on the task manager's internal runAt.
-        // It changes it for timeout, etc when a task is running.
-        startedAt: new Date(Date.now() + alert.interval),
         previousStartedAt: null,
         alertTypeState: {},
         alertInstances: {},

From bd7e2816c16182e84d0356c273c3ae1ec0a76a9c Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 27 Jun 2019 17:28:46 -0400
Subject: [PATCH 10/48] Attempt trying to define custom getBackpressureDelay
 function

---
 x-pack/legacy/plugins/task_manager/task.ts    |  7 +++++++
 .../plugins/task_manager/task_runner.ts       | 14 +++++++++++---
 .../legacy/plugins/task_manager/task_store.ts | 19 +++++++++++++++----
 3 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 494bd59e8460ac..e34775815063bd 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -106,6 +106,12 @@ export interface TaskDefinition {
    */
   maxAttempts?: number;
 
+  /**
+   * Function that returns the delay in miliseconds to wait before attempting the
+   * failed task again.
+   */
+  getBackpressureDelay?: (attempts: number) => number;
+
   /**
    * The numer of workers / slots a running instance of this task occupies.
    * This defaults to 1.
@@ -138,6 +144,7 @@ export const validateTaskDefinition = Joi.object({
     .min(1)
     .default(1),
   createTaskRunner: Joi.func().required(),
+  getBackpressureDelay: Joi.func().optional(),
 }).default();
 
 /**
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index d3f4132fdabb09..32630f07492466 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -212,9 +212,14 @@ export class TaskManagerRunner implements TaskRunner {
 
   private async processResultForRecurringTask(result: RunResult): Promise<RunResult> {
     // recurring task: update the task instance
+    const startedAt = this.instance.startedAt!;
     const state = result.state || this.instance.state || {};
     const maxAttempts = this.definition.maxAttempts || this.store.maxAttempts;
-    const status = this.instance.attempts < maxAttempts ? 'idle' : 'failed';
+    const status = this.instance.interval
+      ? 'idle'
+      : this.instance.attempts < maxAttempts
+      ? 'idle'
+      : 'failed';
 
     let runAt;
     if (status === 'failed') {
@@ -223,9 +228,12 @@ export class TaskManagerRunner implements TaskRunner {
     } else {
       runAt =
         result.runAt ||
-        intervalFromNow(this.instance.interval) ||
         // when result.error is truthy, then we're retrying because it failed
-        minutesFromNow(this.instance.attempts * 5); // incrementally backs off an extra 5m per failure
+        (result.error &&
+          (this.definition.getBackpressureDelay
+            ? new Date(Date.now() + this.definition.getBackpressureDelay(this.instance.attempts))
+            : minutesFromNow(this.instance.attempts * 5))) || // incrementally backs off an extra 5m per failure
+        intervalFromDate(startedAt, this.instance.interval)!;
     }
 
     await this.store.update({
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 37f3a816bb80e3..72e6ced9dbd197 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -305,10 +305,21 @@ export class TaskStore {
                         },
                       },
                       {
-                        range: {
-                          'task.attempts': {
-                            lt: definition.maxAttempts || this.maxAttempts,
-                          },
+                        bool: {
+                          should: [
+                            {
+                              range: {
+                                'task.attempts': {
+                                  lt: definition.maxAttempts || this.maxAttempts,
+                                },
+                              },
+                            },
+                            {
+                              exists: {
+                                field: 'task.interval',
+                              },
+                            },
+                          ],
                         },
                       },
                     ],

From 97811f995735b79b3d45cdac3f2ff5f215ede16e Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 08:22:06 -0400
Subject: [PATCH 11/48] Pass error object to getBackpressureDelay

---
 x-pack/legacy/plugins/task_manager/task.ts        | 2 +-
 x-pack/legacy/plugins/task_manager/task_runner.ts | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index e34775815063bd..9a7d9619bcd516 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -110,7 +110,7 @@ export interface TaskDefinition {
    * Function that returns the delay in miliseconds to wait before attempting the
    * failed task again.
    */
-  getBackpressureDelay?: (attempts: number) => number;
+  getBackpressureDelay?: (attempts: number, error?: object) => number;
 
   /**
    * The numer of workers / slots a running instance of this task occupies.
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 32630f07492466..22924dad58c154 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -231,7 +231,10 @@ export class TaskManagerRunner implements TaskRunner {
         // when result.error is truthy, then we're retrying because it failed
         (result.error &&
           (this.definition.getBackpressureDelay
-            ? new Date(Date.now() + this.definition.getBackpressureDelay(this.instance.attempts))
+            ? new Date(
+                Date.now() +
+                  this.definition.getBackpressureDelay(this.instance.attempts, result.error)
+              )
             : minutesFromNow(this.instance.attempts * 5))) || // incrementally backs off an extra 5m per failure
         intervalFromDate(startedAt, this.instance.interval)!;
     }

From baf939349d1169d6de164292f06a5d16bb5435fc Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 08:42:18 -0400
Subject: [PATCH 12/48] Cleanup processResultForRecurringTask code

---
 .../plugins/task_manager/task_runner.ts       | 37 ++++++++++---------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 22924dad58c154..1efa1ed627bc18 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -214,29 +214,23 @@ export class TaskManagerRunner implements TaskRunner {
     // recurring task: update the task instance
     const startedAt = this.instance.startedAt!;
     const state = result.state || this.instance.state || {};
-    const maxAttempts = this.definition.maxAttempts || this.store.maxAttempts;
-    const status = this.instance.interval
-      ? 'idle'
-      : this.instance.attempts < maxAttempts
-      ? 'idle'
-      : 'failed';
+    const status = this.getInstanceStatus();
 
     let runAt;
     if (status === 'failed') {
       // task run errored, keep the same runAt
       runAt = this.instance.runAt;
+    } else if (result.runAt) {
+      runAt = result.runAt;
+    } else if (result.error) {
+      // when result.error is truthy, then we're retrying because it failed
+      runAt = this.definition.getBackpressureDelay
+        ? new Date(
+            Date.now() + this.definition.getBackpressureDelay(this.instance.attempts, result.error)
+          )
+        : minutesFromNow(this.instance.attempts * 5); // incrementally backs off an extra 5m per failure
     } else {
-      runAt =
-        result.runAt ||
-        // when result.error is truthy, then we're retrying because it failed
-        (result.error &&
-          (this.definition.getBackpressureDelay
-            ? new Date(
-                Date.now() +
-                  this.definition.getBackpressureDelay(this.instance.attempts, result.error)
-              )
-            : minutesFromNow(this.instance.attempts * 5))) || // incrementally backs off an extra 5m per failure
-        intervalFromDate(startedAt, this.instance.interval)!;
+      runAt = intervalFromDate(startedAt, this.instance.interval)!;
     }
 
     await this.store.update({
@@ -276,6 +270,15 @@ export class TaskManagerRunner implements TaskRunner {
     }
     return result;
   }
+
+  private getInstanceStatus() {
+    if (this.instance.interval) {
+      return 'idle';
+    }
+
+    const maxAttempts = this.definition.maxAttempts || this.store.maxAttempts;
+    return this.instance.attempts < maxAttempts ? 'idle' : 'failed';
+  }
 }
 
 function sanitizeInstance(instance: ConcreteTaskInstance): ConcreteTaskInstance {

From fa73ad96f1193c66c4e11bdc934924cb962be6d4 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 10:36:41 -0400
Subject: [PATCH 13/48] Add backpressure to timed out tasks

---
 x-pack/legacy/plugins/task_manager/README.md  |  1 +
 .../task_manager/lib/middleware.test.ts       |  5 +-
 x-pack/legacy/plugins/task_manager/task.ts    | 10 +++
 .../plugins/task_manager/task_runner.test.ts  |  1 +
 .../plugins/task_manager/task_runner.ts       | 10 ++-
 .../plugins/task_manager/task_store.test.ts   |  1 +
 .../legacy/plugins/task_manager/task_store.ts | 65 +++++++++----------
 7 files changed, 56 insertions(+), 37 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/README.md b/x-pack/legacy/plugins/task_manager/README.md
index 0107ac14559a5f..a5534dc61e81d9 100644
--- a/x-pack/legacy/plugins/task_manager/README.md
+++ b/x-pack/legacy/plugins/task_manager/README.md
@@ -20,6 +20,7 @@ At a high-level, the task manager works like this:
 - Attempt to claim the task by using optimistic concurrency to set:
   - status to `running`
   - `startedAt` to now
+  - `retryAt` to next time task should retry if it times out and is still in `running` status
 - Execute the task, if the previous claim succeeded
 - If the task fails, increment the `attempts` count and reschedule it
 - If the task succeeds:
diff --git a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts
index ee82a036c7b444..ff840061285c45 100644
--- a/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/middleware.test.ts
@@ -27,6 +27,7 @@ const getMockConcreteTaskInstance = () => {
     runAt: Date;
     scheduledAt: Date;
     startedAt: Date | null;
+    retryAt: Date | null;
     state: any;
     taskType: string;
     params: any;
@@ -39,6 +40,7 @@ const getMockConcreteTaskInstance = () => {
     runAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()),
     scheduledAt: new Date(moment('2018-09-18T05:33:09.588Z').valueOf()),
     startedAt: null,
+    retryAt: null,
     state: {},
     taskType: 'nice_task',
     params: { abc: 'def' },
@@ -155,10 +157,11 @@ Object {
       "abc": "def",
     },
     "primaryTerm": 1,
+    "retryAt": null,
     "runAt": 2018-09-18T05:33:09.588Z,
     "scheduledAt": 2018-09-18T05:33:09.588Z,
     "sequenceNumber": 1,
-    "startedAt": undefined,
+    "startedAt": null,
     "state": Object {},
     "status": "idle",
     "taskType": "nice_task",
diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 9a7d9619bcd516..84011e337a4510 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -186,6 +186,11 @@ export interface TaskInstance {
    */
   startedAt?: Date | null;
 
+  /**
+   * The date and time that this task should re-execute if stuck in "running" / timeout status.
+   */
+  retryAt?: Date | null;
+
   /**
    * The date and time that this task is scheduled to be run. It is not
    * guaranteed to run at this time, but it is guaranteed not to run earlier
@@ -273,6 +278,11 @@ export interface ConcreteTaskInstance extends TaskInstance {
    */
   startedAt: Date | null;
 
+  /**
+   * The date and time that this task should re-execute if stuck in "running" / timeout status.
+   */
+  retryAt: Date | null;
+
   /**
    * The state passed into the task's run function, and returned by the previous
    * run. If there was no previous run, or if the previous run did not return
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index af7349ad7c906f..0928f7192727b9 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -244,6 +244,7 @@ describe('TaskManagerRunner', () => {
           runAt: new Date(),
           scheduledAt: new Date(),
           startedAt: null,
+          retryAt: null,
           attempts: 0,
           params: {},
           scope: ['reporting'],
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 1efa1ed627bc18..752ea588973e60 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -166,13 +166,18 @@ export class TaskManagerRunner implements TaskRunner {
    */
   public async claimOwnership(): Promise<boolean> {
     const VERSION_CONFLICT_STATUS = 409;
+    const attempts = this.instance.attempts + 1;
+    const now = new Date();
 
     try {
       this.instance = await this.store.update({
         ...this.instance,
         status: 'running',
-        startedAt: new Date(),
-        attempts: this.instance.attempts + 1,
+        startedAt: now,
+        attempts,
+        retryAt: this.definition.getBackpressureDelay
+          ? new Date(now.valueOf() + this.definition.getBackpressureDelay(attempts))
+          : intervalFromDate(now, this.definition.timeout),
       });
 
       return true;
@@ -239,6 +244,7 @@ export class TaskManagerRunner implements TaskRunner {
       state,
       status,
       startedAt: null,
+      retryAt: null,
       attempts: result.error ? this.instance.attempts : 0,
     });
 
diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts
index ef168bd5b3f192..05ca12d23ffca5 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts
@@ -567,6 +567,7 @@ describe('TaskStore', () => {
         runAt,
         scheduledAt: runAt,
         startedAt: null,
+        retryAt: null,
         id: 'task:324242',
         params: { hello: 'world' },
         state: { foo: 'bar' },
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 72e6ced9dbd197..79d6b13b7978d3 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -69,6 +69,7 @@ export interface RawTaskDoc {
       scheduledAt: Date;
       runAt: Date;
       startedAt: Date | null;
+      retryAt: Date | null;
       interval?: string;
       attempts: number;
       status: TaskStatus;
@@ -181,6 +182,7 @@ export class TaskStore {
                 scheduledAt: { type: 'date' },
                 runAt: { type: 'date' },
                 startedAt: { type: 'date' },
+                retryAt: { type: 'date' },
                 interval: { type: 'text' },
                 attempts: { type: 'integer' },
                 status: { type: 'keyword' },
@@ -257,6 +259,7 @@ export class TaskStore {
       status: task.status,
       scheduledAt: task.scheduledAt,
       startedAt: null,
+      retryAt: null,
       runAt: task.runAt,
       state: taskInstance.state || {},
     };
@@ -292,53 +295,46 @@ export class TaskStore {
       query: {
         bool: {
           must: [
-            { range: { 'task.runAt': { lte: 'now' } } },
             { range: { 'kibana.apiVersion': { lte: API_VERSION } } },
             {
               bool: {
-                should: Object.entries(this.definitions).map(([type, definition]) => ({
-                  bool: {
-                    must: [
-                      {
-                        term: {
-                          'task.taskType': type,
-                        },
-                      },
-                      {
-                        bool: {
-                          should: [
-                            {
-                              range: {
-                                'task.attempts': {
-                                  lt: definition.maxAttempts || this.maxAttempts,
-                                },
-                              },
-                            },
-                            {
-                              exists: {
-                                field: 'task.interval',
-                              },
-                            },
-                          ],
-                        },
-                      },
-                    ],
+                should: [
+                  {
+                    bool: {
+                      must: [
+                        { term: { 'task.status': 'idle' } },
+                        { range: { 'task.runAt': { lte: 'now' } } },
+                      ],
+                    },
                   },
-                })),
+                  {
+                    bool: {
+                      must: [
+                        { term: { 'task.status': 'running' } },
+                        { range: { 'task.retryAt': { lte: 'now' } } },
+                      ],
+                    },
+                  },
+                ],
               },
             },
             {
               bool: {
                 should: [
-                  { term: { 'task.status': 'idle' } },
-                  Object.entries(this.definitions).map(([type, definition]) => ({
+                  { exists: { field: 'task.interval' } },
+                  ...Object.entries(this.definitions).map(([type, definition]) => ({
                     bool: {
                       must: [
-                        { term: { 'task.status': 'running' } },
-                        { term: { 'task.taskType': type } },
+                        {
+                          term: {
+                            'task.taskType': type,
+                          },
+                        },
                         {
                           range: {
-                            'task.startedAt': { lte: `now-${definition.timeout}` },
+                            'task.attempts': {
+                              lt: definition.maxAttempts || this.maxAttempts,
+                            },
                           },
                         },
                       ],
@@ -460,6 +456,7 @@ function rawSource(doc: TaskInstance, store: TaskStore) {
     attempts: (doc as ConcreteTaskInstance).attempts || 0,
     scheduledAt: doc.scheduledAt || new Date(),
     startedAt: doc.startedAt || null,
+    retryAt: doc.retryAt || null,
     runAt: doc.runAt || new Date(),
     status: (doc as ConcreteTaskInstance).status || 'idle',
   };

From b20665bd4555dc72b8adcb4b46d36473697cedc8 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 10:40:35 -0400
Subject: [PATCH 14/48] Change default timeout backpressure calculation

---
 x-pack/legacy/plugins/task_manager/task_runner.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 752ea588973e60..1e4f54f1aa87bf 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -177,7 +177,7 @@ export class TaskManagerRunner implements TaskRunner {
         attempts,
         retryAt: this.definition.getBackpressureDelay
           ? new Date(now.valueOf() + this.definition.getBackpressureDelay(attempts))
-          : intervalFromDate(now, this.definition.timeout),
+          : minutesFromNow(attempts * 5), // incrementally backs off an extra 5m per failure
       });
 
       return true;

From e8c2c6f589c75b35cfb9fa0e7b6414f89fb246f5 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 10:56:19 -0400
Subject: [PATCH 15/48] getBackpressureDelay to return seconds instead of
 milliseconds

---
 x-pack/legacy/plugins/task_manager/task.ts        | 2 +-
 x-pack/legacy/plugins/task_manager/task_runner.ts | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 84011e337a4510..6ea00c261f0771 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -107,7 +107,7 @@ export interface TaskDefinition {
   maxAttempts?: number;
 
   /**
-   * Function that returns the delay in miliseconds to wait before attempting the
+   * Function that returns the delay in seconds to wait before attempting the
    * failed task again.
    */
   getBackpressureDelay?: (attempts: number, error?: object) => number;
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 1e4f54f1aa87bf..50e6307db12ea2 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -176,7 +176,7 @@ export class TaskManagerRunner implements TaskRunner {
         startedAt: now,
         attempts,
         retryAt: this.definition.getBackpressureDelay
-          ? new Date(now.valueOf() + this.definition.getBackpressureDelay(attempts))
+          ? new Date(now.valueOf() + this.definition.getBackpressureDelay(attempts) * 1000)
           : minutesFromNow(attempts * 5), // incrementally backs off an extra 5m per failure
       });
 
@@ -231,7 +231,8 @@ export class TaskManagerRunner implements TaskRunner {
       // when result.error is truthy, then we're retrying because it failed
       runAt = this.definition.getBackpressureDelay
         ? new Date(
-            Date.now() + this.definition.getBackpressureDelay(this.instance.attempts, result.error)
+            Date.now() +
+              this.definition.getBackpressureDelay(this.instance.attempts, result.error) * 1000
           )
         : minutesFromNow(this.instance.attempts * 5); // incrementally backs off an extra 5m per failure
     } else {

From 27eabc171f388bcca9d8c1b74553870cb362af85 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 11:01:19 -0400
Subject: [PATCH 16/48] Add comment for task store query

---
 x-pack/legacy/plugins/task_manager/task.ts       | 6 ++++--
 x-pack/legacy/plugins/task_manager/task_store.ts | 3 +++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 6ea00c261f0771..2b86f20a856fc4 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -187,7 +187,8 @@ export interface TaskInstance {
   startedAt?: Date | null;
 
   /**
-   * The date and time that this task should re-execute if stuck in "running" / timeout status.
+   * The date and time that this task should re-execute if stuck in "running" / timeout
+   * status. This value is only set when status is set to "running".
    */
   retryAt?: Date | null;
 
@@ -279,7 +280,8 @@ export interface ConcreteTaskInstance extends TaskInstance {
   startedAt: Date | null;
 
   /**
-   * The date and time that this task should re-execute if stuck in "running" / timeout status.
+   * The date and time that this task should re-execute if stuck in "running" / timeout
+   * status. This value is only set when status is set to "running".
    */
   retryAt: Date | null;
 
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 79d6b13b7978d3..83c4897b840d4d 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -296,6 +296,8 @@ export class TaskStore {
         bool: {
           must: [
             { range: { 'kibana.apiVersion': { lte: API_VERSION } } },
+            // Either a task with idle status and runAt <= now or
+            // status running with a retryAt <= now.
             {
               bool: {
                 should: [
@@ -318,6 +320,7 @@ export class TaskStore {
                 ],
               },
             },
+            // Either task has an interval or the attempts < the maximum configured
             {
               bool: {
                 should: [

From cf7006debc28bc6c445c6a2e9ed6469cc4883a77 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 11:08:38 -0400
Subject: [PATCH 17/48] Compress query

---
 x-pack/legacy/plugins/task_manager/task_store.ts | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 83c4897b840d4d..67635af84dd64f 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -328,11 +328,7 @@ export class TaskStore {
                   ...Object.entries(this.definitions).map(([type, definition]) => ({
                     bool: {
                       must: [
-                        {
-                          term: {
-                            'task.taskType': type,
-                          },
-                        },
+                        { term: { 'task.taskType': type } },
                         {
                           range: {
                             'task.attempts': {

From 5e333bfe858f9ee2a4ddd9eb332e2a431afd5065 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 15:11:56 -0400
Subject: [PATCH 18/48] Revert alert / actions specific code

---
 x-pack/legacy/plugins/alerting/README.md        | 12 ++++++------
 .../alerting/server/alerts_client.test.ts       |  4 +++-
 .../plugins/alerting/server/alerts_client.ts    |  5 ++++-
 .../lib/get_create_task_runner_function.test.ts | 17 +++++------------
 .../lib/get_create_task_runner_function.ts      | 14 ++++++++------
 x-pack/legacy/plugins/alerting/server/types.ts  |  4 ++--
 .../plugins/task_manager/task_runner.test.ts    |  2 +-
 .../legacy/plugins/task_manager/task_runner.ts  |  2 +-
 .../legacy/plugins/task_manager/task_store.ts   |  1 -
 9 files changed, 30 insertions(+), 31 deletions(-)

diff --git a/x-pack/legacy/plugins/alerting/README.md b/x-pack/legacy/plugins/alerting/README.md
index 2f4e265278c604..09ba62e678174d 100644
--- a/x-pack/legacy/plugins/alerting/README.md
+++ b/x-pack/legacy/plugins/alerting/README.md
@@ -52,8 +52,8 @@ This is the primary function for an alert type. Whenever the alert needs to exec
 |services.callCluster(path, opts)|Use this to do Elasticsearch queries on the cluster Kibana connects to. This function is the same as any other `callCluster` in Kibana.<br><br>**NOTE**: This currently authenticates as the Kibana internal user, but will change in a future PR.|
 |services.savedObjectsClient|This is an instance of the saved objects client. This provides the ability to do CRUD on any saved objects within the same space the alert lives in.<br><br>**NOTE**: This currently only works when security is disabled. A future PR will add support for enabled security using Elasticsearch API tokens.|
 |services.log(tags, [data], [timestamp])|Use this to create server logs. (This is the same function as server.log)|
-|startedAt|The date and time the alert type started execution.|
-|previousStartedAt|The previous date and time the alert type started execution.|
+|scheduledRunAt|The date and time the alert type execution was scheduled to be called.|
+|previousScheduledRunAt|The previous date and time the alert type was scheduled to be called.|
 |params|Parameters for the execution. This is where the parameters you require will be passed in. (example threshold). Use alert type validation to ensure values are set before execution.|
 |state|State returned from previous execution. This is the alert level state. What the executor returns will be serialized and provided here at the next execution.|
 
@@ -74,8 +74,8 @@ server.plugins.alerting.registerType({
 			.required(),
 	},
 	async executor({
-		startedAt,
-		previousStartedAt,
+		scheduledRunAt,
+		previousScheduledRunAt,
 		services,
 		params,
 		state,
@@ -133,8 +133,8 @@ server.plugins.alerting.registerType({
 			.required(),
 	},
 	async executor({
-		startedAt,
-		previousStartedAt,
+		scheduledRunAt,
+		previousScheduledRunAt,
 		services,
 		params,
 		state,
diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
index 2b55e24ca67c31..fcc5b22efa0e4a 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
@@ -99,6 +99,7 @@ describe('create()', () => {
       status: 'idle',
       runAt: new Date(),
       startedAt: null,
+      retryAt: null,
       state: {},
       params: {},
     });
@@ -183,7 +184,8 @@ Array [
     "state": Object {
       "alertInstances": Object {},
       "alertTypeState": Object {},
-      "previousStartedAt": null,
+      "previousScheduledRunAt": null,
+      "scheduledRunAt": 2019-02-12T21:01:22.479Z,
     },
     "taskType": "alerting:123",
   },
diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.ts
index 6592f7c3a56722..d1b2c5b9b8ef80 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.ts
@@ -169,7 +169,10 @@ export class AlertsClient {
         basePath,
       },
       state: {
-        previousStartedAt: null,
+        // This is here because we can't rely on the task manager's internal runAt.
+        // It changes it for timeout, etc when a task is running.
+        scheduledRunAt: new Date(Date.now() + alert.interval),
+        previousScheduledRunAt: null,
         alertTypeState: {},
         alertInstances: {},
       },
diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
index f49f438ca463c4..67f8c7a4124e90 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.test.ts
@@ -6,7 +6,6 @@
 
 import Joi from 'joi';
 import { AlertExecutorOptions } from '../types';
-import { ConcreteTaskInstance } from '../../../task_manager';
 import { SavedObjectsClientMock } from '../../../../../../src/core/server/mocks';
 import { getCreateTaskRunnerFunction } from './get_create_task_runner_function';
 
@@ -37,17 +36,10 @@ const getCreateTaskRunnerFunctionParams = {
   internalSavedObjectsRepository: savedObjectsClient,
 };
 
-const mockedTaskInstance: ConcreteTaskInstance = {
-  id: '',
-  attempts: 0,
-  status: 'running',
-  primaryTerm: 0,
-  sequenceNumber: 1,
+const mockedTaskInstance = {
   runAt: mockedLastRunAt,
-  scheduledAt: mockedLastRunAt,
-  startedAt: mockedLastRunAt,
   state: {
-    startedAt: mockedLastRunAt,
+    scheduledRunAt: mockedLastRunAt,
   },
   taskType: 'alerting:test',
   params: {
@@ -96,7 +88,8 @@ Object {
   "state": Object {
     "alertInstances": Object {},
     "alertTypeState": undefined,
-    "previousStartedAt": 2019-06-03T18:55:20.982Z,
+    "previousScheduledRunAt": 2019-06-03T18:55:20.982Z,
+    "scheduledRunAt": 2019-06-03T18:55:30.982Z,
   },
 }
 `);
@@ -107,7 +100,7 @@ Object {
   "bar": true,
 }
 `);
-  expect(call.startedAt).toMatchInlineSnapshot(`2019-06-03T18:55:20.982Z`);
+  expect(call.scheduledRunAt).toMatchInlineSnapshot(`2019-06-03T18:55:20.982Z`);
   expect(call.state).toMatchInlineSnapshot(`Object {}`);
   expect(call.services.alertInstanceFactory).toBeTruthy();
   expect(call.services.callCluster).toBeTruthy();
diff --git a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
index 5591b63188b35b..c21ddbe7ed9863 100644
--- a/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
+++ b/x-pack/legacy/plugins/alerting/server/lib/get_create_task_runner_function.ts
@@ -7,7 +7,7 @@
 import { SavedObjectsClientContract } from 'src/core/server';
 import { ActionsPlugin } from '../../../actions';
 import { AlertType, Services, AlertServices } from '../types';
-import { ConcreteTaskInstance } from '../../../task_manager';
+import { TaskInstance } from '../../../task_manager';
 import { createFireHandler } from './create_fire_handler';
 import { createAlertInstanceFactory } from './create_alert_instance_factory';
 import { AlertInstance } from './alert_instance';
@@ -22,7 +22,7 @@ interface CreateTaskRunnerFunctionOptions {
 }
 
 interface TaskRunnerOptions {
-  taskInstance: ConcreteTaskInstance;
+  taskInstance: TaskInstance;
 }
 
 export function getCreateTaskRunnerFunction({
@@ -66,8 +66,8 @@ export function getCreateTaskRunnerFunction({
           services: alertTypeServices,
           params: validatedAlertTypeParams,
           state: taskInstance.state.alertTypeState || {},
-          startedAt: taskInstance.startedAt!,
-          previousStartedAt: taskInstance.state.previousStartedAt,
+          scheduledRunAt: taskInstance.state.scheduledRunAt,
+          previousScheduledRunAt: taskInstance.state.previousScheduledRunAt,
         });
 
         await Promise.all(
@@ -88,7 +88,7 @@ export function getCreateTaskRunnerFunction({
         );
 
         const nextRunAt = getNextRunAt(
-          new Date(taskInstance.startedAt!),
+          new Date(taskInstance.state.scheduledRunAt),
           alertSavedObject.attributes.interval
         );
 
@@ -96,7 +96,9 @@ export function getCreateTaskRunnerFunction({
           state: {
             alertTypeState,
             alertInstances,
-            previousStartedAt: taskInstance.startedAt!,
+            // We store nextRunAt ourselves since task manager changes runAt when executing a task
+            scheduledRunAt: nextRunAt,
+            previousScheduledRunAt: taskInstance.state.scheduledRunAt,
           },
           runAt: nextRunAt,
         };
diff --git a/x-pack/legacy/plugins/alerting/server/types.ts b/x-pack/legacy/plugins/alerting/server/types.ts
index 9a8f0672ebc9d1..c6e85ba6149926 100644
--- a/x-pack/legacy/plugins/alerting/server/types.ts
+++ b/x-pack/legacy/plugins/alerting/server/types.ts
@@ -29,8 +29,8 @@ export interface AlertServices extends Services {
 }
 
 export interface AlertExecutorOptions {
-  startedAt: Date;
-  previousStartedAt?: Date;
+  scheduledRunAt: Date;
+  previousScheduledRunAt?: Date;
   services: AlertServices;
   params: Record<string, any>;
   state: State;
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 0928f7192727b9..59e35808ca917e 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -88,7 +88,7 @@ describe('TaskManagerRunner', () => {
             },
           }),
         },
-      }
+      },
     });
 
     await runner.run();
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 50e6307db12ea2..68df556a3fc6a9 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -11,7 +11,7 @@
  */
 
 import Joi from 'joi';
-import { intervalFromNow, minutesFromNow, intervalFromDate } from './lib/intervals';
+import { minutesFromNow, intervalFromDate } from './lib/intervals';
 import { Logger } from './lib/logger';
 import { BeforeRunFunction } from './lib/middleware';
 import {
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 67635af84dd64f..51fe2b91a5dbca 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -21,7 +21,6 @@ import {
   TaskInstance,
   TaskStatus,
 } from './task';
-import { intervalFromNow } from './lib/intervals';
 
 export interface StoreOpts {
   callCluster: ElasticJs;

From 89cae1c05802b5d04ec518424b633d95097bec3b Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 15:28:32 -0400
Subject: [PATCH 19/48] Add more interval tests

---
 .../task_manager/lib/intervals.test.ts        | 58 ++++++++++++++++++-
 1 file changed, 57 insertions(+), 1 deletion(-)

diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
index 4552aa9d04b72a..9d0b705592888b 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
@@ -5,7 +5,15 @@
  */
 
 import _ from 'lodash';
-import { assertValidInterval, intervalFromNow, minutesFromNow, secondsFromNow } from './intervals';
+import {
+  assertValidInterval,
+  intervalFromNow,
+  intervalFromDate,
+  minutesFromNow,
+  minutesFromDate,
+  secondsFromNow,
+  secondsFromDate,
+} from './intervals';
 
 describe('taskIntervals', () => {
   describe('assertValidInterval', () => {
@@ -52,6 +60,34 @@ describe('taskIntervals', () => {
     });
   });
 
+  describe('intervalFromDate', () => {
+    test('it returns the given date plus n minutes', () => {
+      const originalDate = new Date(2019, 1, 1);
+      const mins = _.random(1, 100);
+      const expected = originalDate.valueOf() + mins * 60 * 1000;
+      const nextRun = intervalFromDate(originalDate, `${mins}m`)!.getTime();
+      expect(expected).toEqual(nextRun);
+    });
+
+    test('it returns the current date plus n seconds', () => {
+      const originalDate = new Date(2019, 1, 1);
+      const secs = _.random(1, 100);
+      const expected = originalDate.valueOf() + secs * 1000;
+      const nextRun = intervalFromDate(originalDate, `${secs}s`)!.getTime();
+      expect(expected).toEqual(nextRun);
+    });
+
+    test('it rejects intervals are not of the form `Nm` or `Ns`', () => {
+      const date = new Date();
+      expect(() => intervalFromDate(date, `5m 2s`)).toThrow(
+        /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+      expect(() => intervalFromDate(date, `hello`)).toThrow(
+        /Invalid interval "hello"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+    });
+  });
+
   describe('minutesFromNow', () => {
     test('it returns the current date plus a number of minutes', () => {
       const mins = _.random(1, 100);
@@ -61,6 +97,16 @@ describe('taskIntervals', () => {
     });
   });
 
+  describe('minutesFromDate', () => {
+    test('it returns the given date plus a number of minutes', () => {
+      const originalDate = new Date(2019, 1, 1);
+      const mins = _.random(1, 100);
+      const expected = originalDate.valueOf() + mins * 60 * 1000;
+      const nextRun = minutesFromDate(originalDate, mins).getTime();
+      expect(expected).toEqual(nextRun);
+    });
+  });
+
   describe('secondsFromNow', () => {
     test('it returns the current date plus a number of seconds', () => {
       const secs = _.random(1, 100);
@@ -69,4 +115,14 @@ describe('taskIntervals', () => {
       expect(Math.abs(nextRun - expected)).toBeLessThan(100);
     });
   });
+
+  describe('secondsFromDate', () => {
+    test('it returns the given date plus a number of seconds', () => {
+      const originalDate = new Date(2019, 1, 1);
+      const secs = _.random(1, 100);
+      const expected = originalDate.valueOf() + secs * 1000;
+      const nextRun = secondsFromDate(originalDate, secs).getTime();
+      expect(expected).toEqual(nextRun);
+    });
+  });
 });

From cc481a6280d0cc5563b31d82924dcfcadaf46a1a Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 15:41:54 -0400
Subject: [PATCH 20/48] Fix failing jest tests

---
 .../plugins/task_manager/task_runner.test.ts  |  4 +-
 .../plugins/task_manager/task_store.test.ts   | 43 ++++++++++++-------
 2 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 59e35808ca917e..992ed67f37cff4 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -69,7 +69,7 @@ describe('TaskManagerRunner', () => {
     const instance = store.update.args[0][0];
 
     expect(instance.id).toEqual(id);
-    expect(instance.attempts).toEqual(initialAttempts + 1);
+    expect(instance.attempts).toEqual(initialAttempts);
     expect(instance.runAt.getTime()).toBeGreaterThan(Date.now());
     expect(instance.params).toEqual({ a: 'b' });
     expect(instance.state).toEqual({ hey: 'there' });
@@ -79,6 +79,8 @@ describe('TaskManagerRunner', () => {
     const { runner, store } = testOpts({
       instance: {
         interval: '10m',
+        status: 'running',
+        startedAt: new Date(),
       },
       definitions: {
         bar: {
diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts
index 05ca12d23ffca5..9f67d7842f84a5 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts
@@ -426,21 +426,41 @@ describe('TaskStore', () => {
                 {
                   bool: {
                     must: [
+                      { range: { 'kibana.apiVersion': { lte: 1 } } },
                       {
                         bool: {
                           should: [
                             {
                               bool: {
                                 must: [
-                                  {
-                                    term: {
-                                      'task.taskType': 'foo',
-                                    },
-                                  },
+                                  { term: { 'task.status': 'idle' } },
+                                  { range: { 'task.runAt': { lte: 'now' } } },
+                                ],
+                              },
+                            },
+                            {
+                              bool: {
+                                must: [
+                                  { term: { 'task.status': 'running' } },
+                                  { range: { 'task.retryAt': { lte: 'now' } } },
+                                ],
+                              },
+                            },
+                          ],
+                        },
+                      },
+                      {
+                        bool: {
+                          should: [
+                            { exists: { field: 'task.interval' } },
+                            {
+                              bool: {
+                                must: [
+                                  { term: { 'task.taskType': 'foo' } },
                                   {
                                     range: {
                                       'task.attempts': {
-                                        lte: maxAttempts,
+                                        lt: maxAttempts,
                                       },
                                     },
                                   },
@@ -450,15 +470,11 @@ describe('TaskStore', () => {
                             {
                               bool: {
                                 must: [
-                                  {
-                                    term: {
-                                      'task.taskType': 'bar',
-                                    },
-                                  },
+                                  { term: { 'task.taskType': 'bar' } },
                                   {
                                     range: {
                                       'task.attempts': {
-                                        lte: customMaxAttempts,
+                                        lt: customMaxAttempts,
                                       },
                                     },
                                   },
@@ -468,9 +484,6 @@ describe('TaskStore', () => {
                           ],
                         },
                       },
-                      { range: { 'task.runAt': { lte: 'now' } } },
-                      { range: { 'kibana.apiVersion': { lte: 1 } } },
-                      { term: { 'task.status': 'idle' } },
                     ],
                   },
                 },

From fc028a77a55d5417266c25a88318d840541b5a10 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 2 Jul 2019 15:53:32 -0400
Subject: [PATCH 21/48] Fix test

---
 x-pack/legacy/plugins/task_manager/task_runner.test.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 992ed67f37cff4..3717fbdcc13812 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -69,7 +69,6 @@ describe('TaskManagerRunner', () => {
     const instance = store.update.args[0][0];
 
     expect(instance.id).toEqual(id);
-    expect(instance.attempts).toEqual(initialAttempts);
     expect(instance.runAt.getTime()).toBeGreaterThan(Date.now());
     expect(instance.params).toEqual({ a: 'b' });
     expect(instance.state).toEqual({ hey: 'there' });

From 2c20b32a73cdf9414d9fe5822607e228e641d4c0 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 3 Jul 2019 07:45:25 -0400
Subject: [PATCH 22/48] Add more unit tests

---
 x-pack/legacy/plugins/task_manager/task.ts    |   3 +-
 .../plugins/task_manager/task_runner.test.ts  | 103 +++++++++++++++++-
 2 files changed, 103 insertions(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 2b86f20a856fc4..31cfc704a5f8ea 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -102,7 +102,8 @@ export interface TaskDefinition {
   timeout?: string;
 
   /**
-   * Up to how many times the task should retry when it fails to run.
+   * Up to how many times the task should retry when it fails to run. This will
+   * default to the global variable.
    */
   maxAttempts?: number;
 
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 3717fbdcc13812..90c8f40fa444a6 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -6,7 +6,7 @@
 
 import _ from 'lodash';
 import sinon from 'sinon';
-import { minutesFromNow } from './lib/intervals';
+import { minutesFromNow, secondsFromNow } from './lib/intervals';
 import { ConcreteTaskInstance } from './task';
 import { TaskManagerRunner } from './task_runner';
 
@@ -69,7 +69,9 @@ describe('TaskManagerRunner', () => {
     const instance = store.update.args[0][0];
 
     expect(instance.id).toEqual(id);
-    expect(instance.runAt.getTime()).toBeGreaterThan(Date.now());
+    expect(
+      Math.abs(minutesFromNow(initialAttempts * 5).getTime() - instance.runAt.getTime())
+    ).toBeLessThan(100);
     expect(instance.params).toEqual({ a: 'b' });
     expect(instance.state).toEqual({ hey: 'there' });
   });
@@ -212,6 +214,103 @@ describe('TaskManagerRunner', () => {
     sinon.assert.calledWithMatch(logger.warning, /not cancellable/);
   });
 
+  test('sets startedAt, status, attempts and retryAt when claiming a task', async () => {
+    const id = _.random(1, 20).toString();
+    const initialAttempts = _.random(0, 2);
+    const { runner, store } = testOpts({
+      instance: {
+        id,
+        attempts: initialAttempts,
+        interval: undefined,
+      },
+      definitions: {
+        bar: {
+          createTaskRunner: () => ({
+            run: async () => undefined,
+          }),
+        },
+      },
+    });
+
+    await runner.claimOwnership();
+
+    sinon.assert.calledOnce(store.update);
+    const instance = store.update.args[0][0];
+
+    expect(instance.attempts).toEqual(initialAttempts + 1);
+    expect(instance.status).toBe('running');
+    expect(Math.abs(Date.now() - instance.startedAt.getTime())).toBeLessThan(100);
+    expect(
+      Math.abs(minutesFromNow((initialAttempts + 1) * 5).getTime() - instance.retryAt.getTime())
+    ).toBeLessThan(100);
+  });
+
+  test('uses getBackpressureDelay function on error when defined', async () => {
+    const initialAttempts = _.random(0, 2);
+    const backpressureDelay = _.random(15, 100);
+    const id = Date.now().toString();
+    const getBackpressureDelayStub = sinon.stub().returns(backpressureDelay);
+    const error = new Error('Dangit!');
+    const { runner, store } = testOpts({
+      instance: {
+        id,
+        attempts: initialAttempts,
+      },
+      definitions: {
+        bar: {
+          getBackpressureDelay: getBackpressureDelayStub,
+          createTaskRunner: () => ({
+            async run() {
+              throw error;
+            },
+          }),
+        },
+      },
+    });
+
+    await runner.run();
+
+    sinon.assert.calledOnce(store.update);
+    sinon.assert.calledWith(getBackpressureDelayStub, initialAttempts, error);
+    const instance = store.update.args[0][0];
+
+    expect(
+      Math.abs(secondsFromNow(backpressureDelay).getTime() - instance.runAt.getTime())
+    ).toBeLessThan(100);
+  });
+
+  test('uses getBackpressureDelay to set retryAt when defined', async () => {
+    const id = _.random(1, 20).toString();
+    const initialAttempts = _.random(0, 2);
+    const backpressureDelay = _.random(15, 100);
+    const getBackpressureDelayStub = sinon.stub().returns(backpressureDelay);
+    const { runner, store } = testOpts({
+      instance: {
+        id,
+        attempts: initialAttempts,
+        interval: undefined,
+      },
+      definitions: {
+        bar: {
+          getBackpressureDelay: getBackpressureDelayStub,
+          createTaskRunner: () => ({
+            run: async () => undefined,
+          }),
+        },
+      },
+    });
+
+    await runner.claimOwnership();
+
+    sinon.assert.calledOnce(store.update);
+    sinon.assert.calledWith(getBackpressureDelayStub, initialAttempts + 1);
+    const instance = store.update.args[0][0];
+
+    expect(
+      Math.abs(secondsFromNow(backpressureDelay).getTime() - instance.retryAt.getTime())
+    ).toBeLessThan(100);
+  });
+
   interface TestOpts {
     instance?: Partial<ConcreteTaskInstance>;
     definitions?: any;

From 748a8458e898cfb015e7cb796fb810270bc9907f Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 3 Jul 2019 08:43:02 -0400
Subject: [PATCH 23/48] Fix integration tests

---
 .../test_suites/task_manager/task_manager_integration.js        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
index 4bfae4c24a8824..06beb5aa08fc10 100644
--- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
+++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
@@ -106,7 +106,7 @@ export default function ({ getService }) {
         const [scheduledTask] = (await currentTasks()).docs;
         expect(scheduledTask.id).to.eql(task.id);
         expect(scheduledTask.attempts).to.be.greaterThan(0);
-        expect(Date.parse(scheduledTask.runAt)).to.be.greaterThan(Date.parse(task.runAt));
+        expect(Date.parse(scheduledTask.runAt)).to.be.greaterThan(Date.parse(task.runAt) + 5 * 60 * 1000);
       });
     });
 

From 40e043a12aabb8701c13b5ca90d2cfee9d6e63b0 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 3 Jul 2019 09:59:08 -0400
Subject: [PATCH 24/48] Fix sorting of tasks to process

---
 x-pack/legacy/plugins/task_manager/task_store.test.ts | 11 ++++++++++-
 x-pack/legacy/plugins/task_manager/task_store.ts      | 11 ++++++++++-
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts
index 9f67d7842f84a5..302125e4ac90e5 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts
@@ -491,7 +491,16 @@ describe('TaskStore', () => {
             },
           },
           size: 10,
-          sort: { 'task.runAt': { order: 'asc' } },
+          sort: {
+            _script: {
+              type: 'number',
+              order: 'asc',
+              script: {
+                lang: 'expression',
+                source: `doc['task.retryAt'].value || doc['task.runAt'].value`,
+              },
+            },
+          },
           seq_no_primary_term: true,
         },
         index,
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index 51fe2b91a5dbca..8ca3e6e9431267 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -345,7 +345,16 @@ export class TaskStore {
         },
       },
       size: 10,
-      sort: { 'task.runAt': { order: 'asc' } },
+      sort: {
+        _script: {
+          type: 'number',
+          order: 'asc',
+          script: {
+            lang: 'expression',
+            source: `doc['task.retryAt'].value || doc['task.runAt'].value`,
+          },
+        },
+      },
       seq_no_primary_term: true,
     });
 

From c81371372a3a25faa6ea651f8636d76c65be9cf5 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 3 Jul 2019 13:59:26 -0400
Subject: [PATCH 25/48] WIP

---
 .../task_manager/task_manager_integration.js     | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
index 06beb5aa08fc10..f6b3f43b33190f 100644
--- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
+++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
@@ -110,6 +110,22 @@ export default function ({ getService }) {
       });
     });
 
+    it('should fail task if maxAttempts reached', async () => {
+      const task = await scheduleTask({
+        maxAttempts: 1,
+        taskType: 'sampleTask',
+        params: { failWith: 'Dangit!!!!!' },
+      });
+
+      await retry.try(async () => {
+        const [scheduledTask] = (await currentTasks()).docs;
+        expect(scheduledTask.id).to.eql(task.id);
+        expect(scheduledTask.attempts).to.eql(1);
+        expect(scheduledTask.status).to.eql('failed');
+        expect(Date.parse(scheduledTask.runAt)).to.eql(Date.parse(task.runAt));
+      });
+    });
+
     it('should reschedule if task returns runAt', async () => {
       const nextRunMilliseconds = _.random(60000, 200000);
       const count = _.random(1, 20);

From 3b58babbd8ed6e741a01e968792df4a1dd6fee7e Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Mon, 8 Jul 2019 10:45:29 -0400
Subject: [PATCH 26/48] Always provide error when getBackpressureDelay is
 called

---
 x-pack/legacy/plugins/task_manager/task.ts        | 2 +-
 x-pack/legacy/plugins/task_manager/task_runner.ts | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 31cfc704a5f8ea..58816d6fb6e69d 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -111,7 +111,7 @@ export interface TaskDefinition {
    * Function that returns the delay in seconds to wait before attempting the
    * failed task again.
    */
-  getBackpressureDelay?: (attempts: number, error?: object) => number;
+  getBackpressureDelay?: (attempts: number, error: object) => number;
 
   /**
    * The numer of workers / slots a running instance of this task occupies.
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 68df556a3fc6a9..3898a2a0ba1e84 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -11,6 +11,7 @@
  */
 
 import Joi from 'joi';
+import Boom from 'boom';
 import { minutesFromNow, intervalFromDate } from './lib/intervals';
 import { Logger } from './lib/logger';
 import { BeforeRunFunction } from './lib/middleware';
@@ -176,7 +177,10 @@ export class TaskManagerRunner implements TaskRunner {
         startedAt: now,
         attempts,
         retryAt: this.definition.getBackpressureDelay
-          ? new Date(now.valueOf() + this.definition.getBackpressureDelay(attempts) * 1000)
+          ? new Date(
+              now.valueOf() +
+                this.definition.getBackpressureDelay(attempts, Boom.clientTimeout()) * 1000
+            )
           : minutesFromNow(attempts * 5), // incrementally backs off an extra 5m per failure
       });
 

From 7929c8f70f5039d0e8b5b931d4de0ccdd4e51529 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Mon, 8 Jul 2019 15:42:50 -0400
Subject: [PATCH 27/48] Rename getBackpressureDelay to getRetryDelay

---
 x-pack/legacy/plugins/task_manager/task.ts       |  4 ++--
 .../plugins/task_manager/task_runner.test.ts     | 16 ++++++++--------
 .../legacy/plugins/task_manager/task_runner.ts   | 10 ++++------
 3 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index 58816d6fb6e69d..a15be155649a47 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -111,7 +111,7 @@ export interface TaskDefinition {
    * Function that returns the delay in seconds to wait before attempting the
    * failed task again.
    */
-  getBackpressureDelay?: (attempts: number, error: object) => number;
+  getRetryDelay?: (attempts: number, error: object) => number;
 
   /**
    * The numer of workers / slots a running instance of this task occupies.
@@ -145,7 +145,7 @@ export const validateTaskDefinition = Joi.object({
     .min(1)
     .default(1),
   createTaskRunner: Joi.func().required(),
-  getBackpressureDelay: Joi.func().optional(),
+  getRetryDelay: Joi.func().optional(),
 }).default();
 
 /**
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 90c8f40fa444a6..61adc40b301f60 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -245,11 +245,11 @@ describe('TaskManagerRunner', () => {
     ).toBeLessThan(100);
   });
 
-  test('uses getBackpressureDelay function on error when defined', async () => {
+  test('uses getRetryDelay function on error when defined', async () => {
     const initialAttempts = _.random(0, 2);
     const backpressureDelay = _.random(15, 100);
     const id = Date.now().toString();
-    const getBackpressureDelayStub = sinon.stub().returns(backpressureDelay);
+    const getRetryDelayStub = sinon.stub().returns(backpressureDelay);
     const error = new Error('Dangit!');
     const { runner, store } = testOpts({
       instance: {
@@ -258,7 +258,7 @@ describe('TaskManagerRunner', () => {
       },
       definitions: {
         bar: {
-          getBackpressureDelay: getBackpressureDelayStub,
+          getRetryDelay: getRetryDelayStub,
           createTaskRunner: () => ({
             async run() {
               throw error;
@@ -271,7 +271,7 @@ describe('TaskManagerRunner', () => {
     await runner.run();
 
     sinon.assert.calledOnce(store.update);
-    sinon.assert.calledWith(getBackpressureDelayStub, initialAttempts, error);
+    sinon.assert.calledWith(getRetryDelayStub, initialAttempts, error);
     const instance = store.update.args[0][0];
 
     expect(
@@ -279,11 +279,11 @@ describe('TaskManagerRunner', () => {
     ).toBeLessThan(100);
   });
 
-  test('uses getBackpressureDelay to set retryAt when defined', async () => {
+  test('uses getRetryDelay to set retryAt when defined', async () => {
     const id = _.random(1, 20).toString();
     const initialAttempts = _.random(0, 2);
     const backpressureDelay = _.random(15, 100);
-    const getBackpressureDelayStub = sinon.stub().returns(backpressureDelay);
+    const getRetryDelayStub = sinon.stub().returns(backpressureDelay);
     const { runner, store } = testOpts({
       instance: {
         id,
@@ -292,7 +292,7 @@ describe('TaskManagerRunner', () => {
       },
       definitions: {
         bar: {
-          getBackpressureDelay: getBackpressureDelayStub,
+          getRetryDelay: getRetryDelayStub,
           createTaskRunner: () => ({
             run: async () => undefined,
           }),
@@ -303,7 +303,7 @@ describe('TaskManagerRunner', () => {
     await runner.claimOwnership();
 
     sinon.assert.calledOnce(store.update);
-    sinon.assert.calledWith(getBackpressureDelayStub, initialAttempts + 1);
+    sinon.assert.calledWith(getRetryDelayStub, initialAttempts + 1);
     const instance = store.update.args[0][0];
 
     expect(
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 3898a2a0ba1e84..53c0db98b50271 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -176,10 +176,9 @@ export class TaskManagerRunner implements TaskRunner {
         status: 'running',
         startedAt: now,
         attempts,
-        retryAt: this.definition.getBackpressureDelay
+        retryAt: this.definition.getRetryDelay
           ? new Date(
-              now.valueOf() +
-                this.definition.getBackpressureDelay(attempts, Boom.clientTimeout()) * 1000
+              now.valueOf() + this.definition.getRetryDelay(attempts, Boom.clientTimeout()) * 1000
             )
           : minutesFromNow(attempts * 5), // incrementally backs off an extra 5m per failure
       });
@@ -233,10 +232,9 @@ export class TaskManagerRunner implements TaskRunner {
       runAt = result.runAt;
     } else if (result.error) {
       // when result.error is truthy, then we're retrying because it failed
-      runAt = this.definition.getBackpressureDelay
+      runAt = this.definition.getRetryDelay
         ? new Date(
-            Date.now() +
-              this.definition.getBackpressureDelay(this.instance.attempts, result.error) * 1000
+            Date.now() + this.definition.getRetryDelay(this.instance.attempts, result.error) * 1000
           )
         : minutesFromNow(this.instance.attempts * 5); // incrementally backs off an extra 5m per failure
     } else {

From c7f485a0baa7a0f9b180892cfe07bec4cda06857 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Mon, 8 Jul 2019 16:13:06 -0400
Subject: [PATCH 28/48] retryAt to be calculated from timeout time by default

---
 x-pack/legacy/plugins/task_manager/task_runner.ts | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 53c0db98b50271..ae72f1cc216af9 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -12,7 +12,7 @@
 
 import Joi from 'joi';
 import Boom from 'boom';
-import { minutesFromNow, intervalFromDate } from './lib/intervals';
+import { minutesFromNow, intervalFromDate, intervalFromNow } from './lib/intervals';
 import { Logger } from './lib/logger';
 import { BeforeRunFunction } from './lib/middleware';
 import {
@@ -178,7 +178,8 @@ export class TaskManagerRunner implements TaskRunner {
         attempts,
         retryAt: this.definition.getRetryDelay
           ? new Date(
-              now.valueOf() + this.definition.getRetryDelay(attempts, Boom.clientTimeout()) * 1000
+              intervalFromNow(this.definition.timeout!)!.valueOf() +
+                this.definition.getRetryDelay(attempts, Boom.clientTimeout()) * 1000
             )
           : minutesFromNow(attempts * 5), // incrementally backs off an extra 5m per failure
       });

From e0a2436499276a341a2a701cc434d5bb75969154 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 9 Jul 2019 09:08:46 -0400
Subject: [PATCH 29/48] Remove invalid test

---
 .../task_manager/task_manager_integration.js     | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
index f6b3f43b33190f..06beb5aa08fc10 100644
--- a/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
+++ b/x-pack/test/plugin_api_integration/test_suites/task_manager/task_manager_integration.js
@@ -110,22 +110,6 @@ export default function ({ getService }) {
       });
     });
 
-    it('should fail task if maxAttempts reached', async () => {
-      const task = await scheduleTask({
-        maxAttempts: 1,
-        taskType: 'sampleTask',
-        params: { failWith: 'Dangit!!!!!' },
-      });
-
-      await retry.try(async () => {
-        const [scheduledTask] = (await currentTasks()).docs;
-        expect(scheduledTask.id).to.eql(task.id);
-        expect(scheduledTask.attempts).to.eql(1);
-        expect(scheduledTask.status).to.eql('failed');
-        expect(Date.parse(scheduledTask.runAt)).to.eql(Date.parse(task.runAt));
-      });
-    });
-
     it('should reschedule if task returns runAt', async () => {
       const nextRunMilliseconds = _.random(60000, 200000);
       const count = _.random(1, 20);

From 94f07acb6320aee01851471847db30f5c8afa9f2 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 9 Jul 2019 09:25:31 -0400
Subject: [PATCH 30/48] Add unit tests

---
 .../plugins/task_manager/task_runner.test.ts  | 69 ++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 61adc40b301f60..3ab070f06abbf6 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -283,6 +283,7 @@ describe('TaskManagerRunner', () => {
     const id = _.random(1, 20).toString();
     const initialAttempts = _.random(0, 2);
     const backpressureDelay = _.random(15, 100);
+    const timeoutMinutes = 1;
     const getRetryDelayStub = sinon.stub().returns(backpressureDelay);
     const { runner, store } = testOpts({
       instance: {
@@ -292,6 +293,7 @@ describe('TaskManagerRunner', () => {
       },
       definitions: {
         bar: {
+          timeout: `${timeoutMinutes}m`,
           getRetryDelay: getRetryDelayStub,
           createTaskRunner: () => ({
             run: async () => undefined,
@@ -307,10 +309,75 @@ describe('TaskManagerRunner', () => {
     const instance = store.update.args[0][0];
 
     expect(
-      Math.abs(secondsFromNow(backpressureDelay).getTime() - instance.retryAt.getTime())
+      Math.abs(
+        secondsFromNow(backpressureDelay).getTime() +
+          timeoutMinutes * 60 * 1000 -
+          instance.retryAt.getTime()
+      )
     ).toBeLessThan(100);
   });
 
+  test('Fails non-recurring task when maxAttempts reached', async () => {
+    const id = _.random(1, 20).toString();
+    const initialAttempts = 3;
+    const { runner, store } = testOpts({
+      instance: {
+        id,
+        attempts: initialAttempts,
+        interval: undefined,
+      },
+      definitions: {
+        bar: {
+          maxAttempts: 3,
+          createTaskRunner: () => ({
+            run: async () => {
+              throw new Error();
+            },
+          }),
+        },
+      },
+    });
+
+    await runner.run();
+
+    sinon.assert.calledOnce(store.update);
+    const instance = store.update.args[0][0];
+    expect(instance.attempts).toEqual(3);
+    expect(instance.status).toEqual('failed');
+    expect(instance.retryAt).toBeNull();
+    expect(instance.runAt.getTime()).toBeLessThanOrEqual(Date.now());
+  });
+
+  test(`Doesn't fail recurring tasks when maxAttempts reached`, async () => {
+    const id = _.random(1, 20).toString();
+    const initialAttempts = 3;
+    const { runner, store } = testOpts({
+      instance: {
+        id,
+        attempts: initialAttempts,
+        interval: '10s',
+      },
+      definitions: {
+        bar: {
+          maxAttempts: 3,
+          createTaskRunner: () => ({
+            run: async () => {
+              throw new Error();
+            },
+          }),
+        },
+      },
+    });
+
+    await runner.run();
+
+    sinon.assert.calledOnce(store.update);
+    const instance = store.update.args[0][0];
+    expect(instance.attempts).toEqual(3);
+    expect(instance.status).toEqual('idle');
+    expect(Math.abs(minutesFromNow(15).getTime() - instance.runAt.getTime())).toBeLessThan(100);
+  });
+
   interface TestOpts {
     instance?: Partial<ConcreteTaskInstance>;
     definitions?: any;

From 3f28c2dc1eaa8e5cb5f33d87eb758e052ee758d9 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 9 Jul 2019 09:36:30 -0400
Subject: [PATCH 31/48] Consider timeout before scheduling a retryAt

---
 .../plugins/task_manager/task_runner.test.ts       |  8 +++++++-
 x-pack/legacy/plugins/task_manager/task_runner.ts  | 14 ++++++++------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 3ab070f06abbf6..3f9ff060221b81 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -215,6 +215,7 @@ describe('TaskManagerRunner', () => {
   });
 
   test('sets startedAt, status, attempts and retryAt when claiming a task', async () => {
+    const timeoutMinutes = 1;
     const id = _.random(1, 20).toString();
     const initialAttempts = _.random(0, 2);
     const { runner, store } = testOpts({
@@ -225,6 +226,7 @@ describe('TaskManagerRunner', () => {
       },
       definitions: {
         bar: {
+          timeout: `${timeoutMinutes}m`,
           createTaskRunner: () => ({
             run: async () => undefined,
           }),
@@ -241,7 +243,11 @@ describe('TaskManagerRunner', () => {
     expect(instance.status).toBe('running');
     expect(Math.abs(Date.now() - instance.startedAt.getTime())).toBeLessThan(100);
     expect(
-      Math.abs(minutesFromNow((initialAttempts + 1) * 5).getTime() - instance.retryAt.getTime())
+      Math.abs(
+        minutesFromNow((initialAttempts + 1) * 5).getTime() +
+          timeoutMinutes * 60 * 1000 -
+          instance.retryAt.getTime()
+      )
     ).toBeLessThan(100);
   });
 
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index ae72f1cc216af9..1552f3660dc628 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -170,18 +170,20 @@ export class TaskManagerRunner implements TaskRunner {
     const attempts = this.instance.attempts + 1;
     const now = new Date();
 
+    const timeoutDate = intervalFromNow(this.definition.timeout!)!;
+
     try {
       this.instance = await this.store.update({
         ...this.instance,
         status: 'running',
         startedAt: now,
         attempts,
-        retryAt: this.definition.getRetryDelay
-          ? new Date(
-              intervalFromNow(this.definition.timeout!)!.valueOf() +
-                this.definition.getRetryDelay(attempts, Boom.clientTimeout()) * 1000
-            )
-          : minutesFromNow(attempts * 5), // incrementally backs off an extra 5m per failure
+        retryAt: new Date(
+          timeoutDate.getTime() +
+            (this.definition.getRetryDelay
+              ? this.definition.getRetryDelay(attempts, Boom.clientTimeout()) * 1000
+              : attempts * 5 * 60 * 1000) // incrementally backs off an extra 5m per failure
+        ),
       });
 
       return true;

From 3ad1b797ccd1fb03294944c95ae12f3b22da79e0 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 9 Jul 2019 09:41:43 -0400
Subject: [PATCH 32/48] Remove backpressure terminology

---
 .../plugins/task_manager/task_runner.test.ts     | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 3f9ff060221b81..dcd41b42102de5 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -253,9 +253,9 @@ describe('TaskManagerRunner', () => {
 
   test('uses getRetryDelay function on error when defined', async () => {
     const initialAttempts = _.random(0, 2);
-    const backpressureDelay = _.random(15, 100);
+    const retryDelay = _.random(15, 100);
     const id = Date.now().toString();
-    const getRetryDelayStub = sinon.stub().returns(backpressureDelay);
+    const getRetryDelayStub = sinon.stub().returns(retryDelay);
     const error = new Error('Dangit!');
     const { runner, store } = testOpts({
       instance: {
@@ -280,17 +280,17 @@ describe('TaskManagerRunner', () => {
     sinon.assert.calledWith(getRetryDelayStub, initialAttempts, error);
     const instance = store.update.args[0][0];
 
-    expect(
-      Math.abs(secondsFromNow(backpressureDelay).getTime() - instance.runAt.getTime())
-    ).toBeLessThan(100);
+    expect(Math.abs(secondsFromNow(retryDelay).getTime() - instance.runAt.getTime())).toBeLessThan(
+      100
+    );
   });
 
   test('uses getRetryDelay to set retryAt when defined', async () => {
     const id = _.random(1, 20).toString();
     const initialAttempts = _.random(0, 2);
-    const backpressureDelay = _.random(15, 100);
+    const retryDelay = _.random(15, 100);
     const timeoutMinutes = 1;
-    const getRetryDelayStub = sinon.stub().returns(backpressureDelay);
+    const getRetryDelayStub = sinon.stub().returns(retryDelay);
     const { runner, store } = testOpts({
       instance: {
         id,
@@ -316,7 +316,7 @@ describe('TaskManagerRunner', () => {
 
     expect(
       Math.abs(
-        secondsFromNow(backpressureDelay).getTime() +
+        secondsFromNow(retryDelay).getTime() +
           timeoutMinutes * 60 * 1000 -
           instance.retryAt.getTime()
       )

From f08ccb63a0104a94c27c249a2f94bf2dd24205db Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 9 Jul 2019 09:50:42 -0400
Subject: [PATCH 33/48] Remove support for 0 based intervals and timeouts

---
 .../task_manager/lib/intervals.test.ts        | 28 +++++++++++++++++++
 .../plugins/task_manager/lib/intervals.ts     |  4 +--
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
index 9d0b705592888b..bff8b8f755b8b5 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
@@ -25,6 +25,15 @@ describe('taskIntervals', () => {
       expect(() => assertValidInterval(`${_.random(1000)}s`)).not.toThrow();
     });
 
+    test('it rejects 0 based intervals', () => {
+      expect(() => assertValidInterval('0m')).toThrow(
+        /Invalid interval "0m"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+      expect(() => assertValidInterval('0s')).toThrow(
+        /Invalid interval "0s"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+    });
+
     test('it rejects intervals are not of the form `Nm` or `Ns`', () => {
       expect(() => assertValidInterval(`5m 2s`)).toThrow(
         /Invalid interval "5m 2s"\. Intervals must be of the form {number}m. Example: 5m/
@@ -58,6 +67,15 @@ describe('taskIntervals', () => {
         /Invalid interval "hello"\. Intervals must be of the form {number}m. Example: 5m/
       );
     });
+
+    test('it rejects 0 based intervals', () => {
+      expect(() => intervalFromNow('0m')).toThrow(
+        /Invalid interval "0m"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+      expect(() => intervalFromNow('0s')).toThrow(
+        /Invalid interval "0s"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+    });
   });
 
   describe('intervalFromDate', () => {
@@ -86,6 +104,16 @@ describe('taskIntervals', () => {
         /Invalid interval "hello"\. Intervals must be of the form {number}m. Example: 5m/
       );
     });
+
+    test('it rejects 0 based intervals', () => {
+      const date = new Date();
+      expect(() => intervalFromDate(date, '0m')).toThrow(
+        /Invalid interval "0m"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+      expect(() => intervalFromDate(date, '0s')).toThrow(
+        /Invalid interval "0s"\. Intervals must be of the form {number}m. Example: 5m/
+      );
+    });
   });
 
   describe('minutesFromNow', () => {
diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
index 0d210a413de2d1..f4cdb20cdaa2d7 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
@@ -123,9 +123,9 @@ function parseInterval(interval: string) {
 }
 
 function isMinutes(interval: string) {
-  return /^[0-9]+m$/.test(interval);
+  return /^[1-9][0-9]*m$/.test(interval);
 }
 
 function isSeconds(interval: string) {
-  return /^[0-9]+s$/.test(interval);
+  return /^[1-9][0-9]*s$/.test(interval);
 }

From 308b58294d2bac2d18c00ea0f76b3affc1b16e03 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 9 Jul 2019 13:18:27 -0400
Subject: [PATCH 34/48] Apply PR feedback

---
 .../task_manager/lib/intervals.test.ts        | 18 ++++++--
 .../plugins/task_manager/lib/intervals.ts     | 12 +-----
 .../plugins/task_manager/task_runner.test.ts  | 41 +++++++++----------
 .../plugins/task_manager/task_runner.ts       |  2 +-
 4 files changed, 36 insertions(+), 37 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
index bff8b8f755b8b5..7e8b2dbac69bf4 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
@@ -15,6 +15,16 @@ import {
   secondsFromDate,
 } from './intervals';
 
+const mockedNow = new Date('2019-06-03T18:55:25.982Z');
+(global as any).Date = class Date extends global.Date {
+  static now() {
+    return mockedNow.getTime();
+  }
+  valueOf() {
+    return mockedNow.valueOf();
+  }
+};
+
 describe('taskIntervals', () => {
   describe('assertValidInterval', () => {
     test('it accepts intervals in the form `Nm`', () => {
@@ -49,14 +59,14 @@ describe('taskIntervals', () => {
       const mins = _.random(1, 100);
       const expected = Date.now() + mins * 60 * 1000;
       const nextRun = intervalFromNow(`${mins}m`)!.getTime();
-      expect(Math.abs(nextRun - expected)).toBeLessThan(100);
+      expect(nextRun).toEqual(expected);
     });
 
     test('it returns the current date plus n seconds', () => {
       const secs = _.random(1, 100);
       const expected = Date.now() + secs * 1000;
       const nextRun = intervalFromNow(`${secs}s`)!.getTime();
-      expect(Math.abs(nextRun - expected)).toBeLessThan(100);
+      expect(nextRun).toEqual(expected);
     });
 
     test('it rejects intervals are not of the form `Nm` or `Ns`', () => {
@@ -121,7 +131,7 @@ describe('taskIntervals', () => {
       const mins = _.random(1, 100);
       const expected = Date.now() + mins * 60 * 1000;
       const nextRun = minutesFromNow(mins).getTime();
-      expect(Math.abs(nextRun - expected)).toBeLessThan(100);
+      expect(nextRun).toEqual(expected);
     });
   });
 
@@ -140,7 +150,7 @@ describe('taskIntervals', () => {
       const secs = _.random(1, 100);
       const expected = Date.now() + secs * 1000;
       const nextRun = secondsFromNow(secs).getTime();
-      expect(Math.abs(nextRun - expected)).toBeLessThan(100);
+      expect(nextRun).toEqual(expected);
     });
   });
 
diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
index f4cdb20cdaa2d7..92b68a7e6a7596 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
@@ -51,11 +51,7 @@ export function intervalFromDate(date: Date, interval?: string): Date | undefine
  * @param mins The number of mintues from now
  */
 export function minutesFromNow(mins: number): Date {
-  const now = new Date();
-
-  now.setMinutes(now.getMinutes() + mins);
-
-  return now;
+  return minutesFromDate(new Date(Date.now()), mins);
 }
 
 /**
@@ -78,11 +74,7 @@ export function minutesFromDate(date: Date, mins: number): Date {
  * @param secs The number of seconds from now
  */
 export function secondsFromNow(secs: number): Date {
-  const now = new Date();
-
-  now.setSeconds(now.getSeconds() + secs);
-
-  return now;
+  return secondsFromDate(new Date(Date.now()), secs);
 }
 
 /**
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index dcd41b42102de5..67096c01b64798 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -10,6 +10,13 @@ import { minutesFromNow, secondsFromNow } from './lib/intervals';
 import { ConcreteTaskInstance } from './task';
 import { TaskManagerRunner } from './task_runner';
 
+const mockedNow = new Date('2019-06-03T18:55:25.982Z');
+(global as any).Date = class Date extends global.Date {
+  static now() {
+    return mockedNow.getTime();
+  }
+};
+
 describe('TaskManagerRunner', () => {
   test('provides details about the task that is running', () => {
     const { runner } = testOpts({
@@ -69,9 +76,7 @@ describe('TaskManagerRunner', () => {
     const instance = store.update.args[0][0];
 
     expect(instance.id).toEqual(id);
-    expect(
-      Math.abs(minutesFromNow(initialAttempts * 5).getTime() - instance.runAt.getTime())
-    ).toBeLessThan(100);
+    expect(instance.runAt.getTime()).toEqual(minutesFromNow(initialAttempts * 5).getTime());
     expect(instance.params).toEqual({ a: 'b' });
     expect(instance.state).toEqual({ hey: 'there' });
   });
@@ -81,7 +86,7 @@ describe('TaskManagerRunner', () => {
       instance: {
         interval: '10m',
         status: 'running',
-        startedAt: new Date(),
+        startedAt: new Date(Date.now()),
       },
       definitions: {
         bar: {
@@ -241,14 +246,10 @@ describe('TaskManagerRunner', () => {
 
     expect(instance.attempts).toEqual(initialAttempts + 1);
     expect(instance.status).toBe('running');
-    expect(Math.abs(Date.now() - instance.startedAt.getTime())).toBeLessThan(100);
-    expect(
-      Math.abs(
-        minutesFromNow((initialAttempts + 1) * 5).getTime() +
-          timeoutMinutes * 60 * 1000 -
-          instance.retryAt.getTime()
-      )
-    ).toBeLessThan(100);
+    expect(instance.startedAt.getTime()).toEqual(Date.now());
+    expect(instance.retryAt.getTime()).toEqual(
+      minutesFromNow((initialAttempts + 1) * 5).getTime() + timeoutMinutes * 60 * 1000
+    );
   });
 
   test('uses getRetryDelay function on error when defined', async () => {
@@ -314,13 +315,9 @@ describe('TaskManagerRunner', () => {
     sinon.assert.calledWith(getRetryDelayStub, initialAttempts + 1);
     const instance = store.update.args[0][0];
 
-    expect(
-      Math.abs(
-        secondsFromNow(retryDelay).getTime() +
-          timeoutMinutes * 60 * 1000 -
-          instance.retryAt.getTime()
-      )
-    ).toBeLessThan(100);
+    expect(instance.retryAt.getTime()).toEqual(
+      secondsFromNow(retryDelay).getTime() + timeoutMinutes * 60 * 1000
+    );
   });
 
   test('Fails non-recurring task when maxAttempts reached', async () => {
@@ -381,7 +378,7 @@ describe('TaskManagerRunner', () => {
     const instance = store.update.args[0][0];
     expect(instance.attempts).toEqual(3);
     expect(instance.status).toEqual('idle');
-    expect(Math.abs(minutesFromNow(15).getTime() - instance.runAt.getTime())).toBeLessThan(100);
+    expect(instance.runAt.getTime()).toEqual(minutesFromNow(15).getTime());
   });
 
   interface TestOpts {
@@ -414,8 +411,8 @@ describe('TaskManagerRunner', () => {
           taskType: 'bar',
           sequenceNumber: 32,
           primaryTerm: 32,
-          runAt: new Date(),
-          scheduledAt: new Date(),
+          runAt: new Date(Date.now()),
+          scheduledAt: new Date(Date.now()),
           startedAt: null,
           retryAt: null,
           attempts: 0,
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 1552f3660dc628..4387a54e65f489 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -168,7 +168,7 @@ export class TaskManagerRunner implements TaskRunner {
   public async claimOwnership(): Promise<boolean> {
     const VERSION_CONFLICT_STATUS = 409;
     const attempts = this.instance.attempts + 1;
-    const now = new Date();
+    const now = new Date(Date.now());
 
     const timeoutDate = intervalFromNow(this.definition.timeout!)!;
 

From c54901d3f27395fb94fb1be17d864b5e677e9695 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 9 Jul 2019 13:20:02 -0400
Subject: [PATCH 35/48] Fix last place using Math.abs

---
 x-pack/legacy/plugins/task_manager/task_runner.test.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 67096c01b64798..af5e864edf6b42 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -281,9 +281,7 @@ describe('TaskManagerRunner', () => {
     sinon.assert.calledWith(getRetryDelayStub, initialAttempts, error);
     const instance = store.update.args[0][0];
 
-    expect(Math.abs(secondsFromNow(retryDelay).getTime() - instance.runAt.getTime())).toBeLessThan(
-      100
-    );
+    expect(instance.runAt.getTime()).toEqual(secondsFromNow(retryDelay).getTime());
   });
 
   test('uses getRetryDelay to set retryAt when defined', async () => {

From 6c07dcfb3fb5764de5a2911c22b776b97c12b6db Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 16 Jul 2019 09:09:02 -0400
Subject: [PATCH 36/48] Modify migrations to allow running a script when
 converting an index to an alias

---
 .../migrations/core/build_index_map.test.ts   | 158 ++++++++++++++++++
 .../migrations/core/build_index_map.ts        |  22 ++-
 .../migrations/core/call_cluster.ts           |   4 +
 .../migrations/core/elastic_index.test.ts     |  12 +-
 .../migrations/core/elastic_index.ts          |  19 ++-
 .../migrations/core/index_migrator.ts         |   2 +-
 .../migrations/core/migration_context.ts      |   3 +
 .../migrations/kibana/kibana_migrator.ts      |   3 +-
 .../server/saved_objects/schema/schema.ts     |   1 +
 9 files changed, 214 insertions(+), 10 deletions(-)
 create mode 100644 src/core/server/saved_objects/migrations/core/build_index_map.test.ts

diff --git a/src/core/server/saved_objects/migrations/core/build_index_map.test.ts b/src/core/server/saved_objects/migrations/core/build_index_map.test.ts
new file mode 100644
index 00000000000000..afa44071d93de1
--- /dev/null
+++ b/src/core/server/saved_objects/migrations/core/build_index_map.test.ts
@@ -0,0 +1,158 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createIndexMap } from './build_index_map';
+
+test('mappings without index pattern goes to default index', () => {
+  const result = createIndexMap(
+    '.kibana',
+    {
+      type1: {
+        isNamespaceAgnostic: false,
+      },
+    },
+    {
+      type1: {
+        properties: {
+          field1: {
+            type: 'string',
+          },
+        },
+      },
+    }
+  );
+  expect(result).toEqual({
+    '.kibana': {
+      typeMappings: {
+        type1: {
+          properties: {
+            field1: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+  });
+});
+
+test(`mappings with custom index pattern doesn't go to default index`, () => {
+  const result = createIndexMap(
+    '.kibana',
+    {
+      type1: {
+        isNamespaceAgnostic: false,
+        indexPattern: '.other_kibana',
+      },
+    },
+    {
+      type1: {
+        properties: {
+          field1: {
+            type: 'string',
+          },
+        },
+      },
+    }
+  );
+  expect(result).toEqual({
+    '.other_kibana': {
+      typeMappings: {
+        type1: {
+          properties: {
+            field1: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+  });
+});
+
+test('creating a script gets added to the index pattern', () => {
+  const result = createIndexMap(
+    '.kibana',
+    {
+      type1: {
+        isNamespaceAgnostic: false,
+        indexPattern: '.other_kibana',
+        convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
+      },
+    },
+    {
+      type1: {
+        properties: {
+          field1: {
+            type: 'string',
+          },
+        },
+      },
+    }
+  );
+  expect(result).toEqual({
+    '.other_kibana': {
+      script: `ctx._id = ctx._source.type + ':' + ctx._id`,
+      typeMappings: {
+        type1: {
+          properties: {
+            field1: {
+              type: 'string',
+            },
+          },
+        },
+      },
+    },
+  });
+});
+
+test('throws when two scripts are defined for an index pattern', () => {
+  const defaultIndex = '.kibana';
+  const savedObjectSchemas = {
+    type1: {
+      isNamespaceAgnostic: false,
+      convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
+    },
+    type2: {
+      isNamespaceAgnostic: false,
+      convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
+    },
+  };
+  const indexMap = {
+    type1: {
+      properties: {
+        field1: {
+          type: 'string',
+        },
+      },
+    },
+    type2: {
+      properties: {
+        field1: {
+          type: 'string',
+        },
+      },
+    },
+  };
+  expect(() =>
+    createIndexMap(defaultIndex, savedObjectSchemas, indexMap)
+  ).toThrowErrorMatchingInlineSnapshot(
+    `"A script has been defined more than once for index pattern \\".kibana\\""`
+  );
+});
diff --git a/src/core/server/saved_objects/migrations/core/build_index_map.ts b/src/core/server/saved_objects/migrations/core/build_index_map.ts
index 365c79692ba0d9..7d146f8b8e4a0a 100644
--- a/src/core/server/saved_objects/migrations/core/build_index_map.ts
+++ b/src/core/server/saved_objects/migrations/core/build_index_map.ts
@@ -20,6 +20,13 @@
 import { MappingProperties } from '../../mappings';
 import { SavedObjectsSchemaDefinition } from '../../schema';
 
+export interface IndexMap {
+  [index: string]: {
+    typeMappings: MappingProperties;
+    script?: string;
+  };
+}
+
 /*
  * This file contains logic to convert savedObjectSchemas into a dictonary of indexes and documents
  */
@@ -28,13 +35,20 @@ export function createIndexMap(
   savedObjectSchemas: SavedObjectsSchemaDefinition,
   indexMap: MappingProperties
 ) {
-  const map: { [index: string]: MappingProperties } = {};
+  const map: IndexMap = {};
   Object.keys(indexMap).forEach(type => {
-    const indexPattern = (savedObjectSchemas[type] || {}).indexPattern || defaultIndex;
+    const schema = savedObjectSchemas[type] || {};
+    const script = schema.convertToAliasScript;
+    const indexPattern = schema.indexPattern || defaultIndex;
     if (!map.hasOwnProperty(indexPattern as string)) {
-      map[indexPattern] = {};
+      map[indexPattern] = { typeMappings: {} };
+    }
+    map[indexPattern].typeMappings[type] = indexMap[type];
+    if (script && map[indexPattern].script) {
+      throw Error(`A script has been defined more than once for index pattern "${indexPattern}"`);
+    } else if (script) {
+      map[indexPattern].script = script;
     }
-    map[indexPattern][type] = indexMap[type];
   });
   return map;
 }
diff --git a/src/core/server/saved_objects/migrations/core/call_cluster.ts b/src/core/server/saved_objects/migrations/core/call_cluster.ts
index f5b4f787a61d4d..628f2785e6c649 100644
--- a/src/core/server/saved_objects/migrations/core/call_cluster.ts
+++ b/src/core/server/saved_objects/migrations/core/call_cluster.ts
@@ -90,6 +90,10 @@ export interface ReindexOpts {
   body: {
     dest: IndexOpts;
     source: IndexOpts & { size: number };
+    script?: {
+      source: string;
+      lang: 'painless';
+    };
   };
   refresh: boolean;
   waitForCompletion: boolean;
diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts
index 65df2fd580d84f..393cbb7fbb2aef 100644
--- a/src/core/server/saved_objects/migrations/core/elastic_index.test.ts
+++ b/src/core/server/saved_objects/migrations/core/elastic_index.test.ts
@@ -231,6 +231,10 @@ describe('ElasticIndex', () => {
               body: {
                 dest: { index: '.ze-index' },
                 source: { index: '.muchacha' },
+                script: {
+                  source: `ctx._id = ctx._source.type + ':' + ctx._id`,
+                  lang: 'painless',
+                },
               },
               refresh: true,
               waitForCompletion: false,
@@ -267,7 +271,13 @@ describe('ElasticIndex', () => {
           properties: { foo: { type: 'keyword' } },
         },
       };
-      await Index.convertToAlias(callCluster as any, info, '.muchacha', 10);
+      await Index.convertToAlias(
+        callCluster as any,
+        info,
+        '.muchacha',
+        10,
+        `ctx._id = ctx._source.type + ':' + ctx._id`
+      );
 
       expect(callCluster.mock.calls.map(([path]) => path)).toEqual([
         'indices.create',
diff --git a/src/core/server/saved_objects/migrations/core/elastic_index.ts b/src/core/server/saved_objects/migrations/core/elastic_index.ts
index 9606a46edef959..da76905d1c65c6 100644
--- a/src/core/server/saved_objects/migrations/core/elastic_index.ts
+++ b/src/core/server/saved_objects/migrations/core/elastic_index.ts
@@ -228,14 +228,15 @@ export async function convertToAlias(
   callCluster: CallCluster,
   info: FullIndexInfo,
   alias: string,
-  batchSize: number
+  batchSize: number,
+  script?: string
 ) {
   await callCluster('indices.create', {
     body: { mappings: info.mappings, settings },
     index: info.indexName,
   });
 
-  await reindex(callCluster, alias, info.indexName, batchSize);
+  await reindex(callCluster, alias, info.indexName, batchSize, script);
 
   await claimAlias(callCluster, info.indexName, alias, [{ remove_index: { index: alias } }]);
 }
@@ -316,7 +317,13 @@ function assertResponseIncludeAllShards({ _shards }: { _shards: ShardsInfo }) {
 /**
  * Reindexes from source to dest, polling for the reindex completion.
  */
-async function reindex(callCluster: CallCluster, source: string, dest: string, batchSize: number) {
+async function reindex(
+  callCluster: CallCluster,
+  source: string,
+  dest: string,
+  batchSize: number,
+  script?: string
+) {
   // We poll instead of having the request wait for completion, as for large indices,
   // the request times out on the Elasticsearch side of things. We have a relatively tight
   // polling interval, as the request is fairly efficent, and we don't
@@ -326,6 +333,12 @@ async function reindex(callCluster: CallCluster, source: string, dest: string, b
     body: {
       dest: { index: dest },
       source: { index: source, size: batchSize },
+      script: script
+        ? {
+            source: script,
+            lang: 'painless',
+          }
+        : undefined,
     },
     refresh: true,
     waitForCompletion: false,
diff --git a/src/core/server/saved_objects/migrations/core/index_migrator.ts b/src/core/server/saved_objects/migrations/core/index_migrator.ts
index 7fc2bcfb72602e..c75fa68572c710 100644
--- a/src/core/server/saved_objects/migrations/core/index_migrator.ts
+++ b/src/core/server/saved_objects/migrations/core/index_migrator.ts
@@ -176,7 +176,7 @@ async function migrateSourceToDest(context: Context) {
   if (!source.aliases[alias]) {
     log.info(`Reindexing ${alias} to ${source.indexName}`);
 
-    await Index.convertToAlias(callCluster, source, alias, batchSize);
+    await Index.convertToAlias(callCluster, source, alias, batchSize, context.convertToAliasScript);
   }
 
   const read = Index.reader(callCluster, source.indexName, { batchSize, scrollDuration });
diff --git a/src/core/server/saved_objects/migrations/core/migration_context.ts b/src/core/server/saved_objects/migrations/core/migration_context.ts
index f3c4b271c3a72c..a151a8d37a524c 100644
--- a/src/core/server/saved_objects/migrations/core/migration_context.ts
+++ b/src/core/server/saved_objects/migrations/core/migration_context.ts
@@ -42,6 +42,7 @@ export interface MigrationOpts {
   mappingProperties: MappingProperties;
   documentMigrator: VersionedTransformer;
   serializer: SavedObjectsSerializer;
+  convertToAliasScript?: string;
 
   /**
    * If specified, templates matching the specified pattern will be removed
@@ -62,6 +63,7 @@ export interface Context {
   scrollDuration: string;
   serializer: SavedObjectsSerializer;
   obsoleteIndexTemplatePattern?: string;
+  convertToAliasScript?: string;
 }
 
 /**
@@ -87,6 +89,7 @@ export async function migrationContext(opts: MigrationOpts): Promise<Context> {
     scrollDuration: opts.scrollDuration,
     serializer: opts.serializer,
     obsoleteIndexTemplatePattern: opts.obsoleteIndexTemplatePattern,
+    convertToAliasScript: opts.convertToAliasScript,
   };
 }
 
diff --git a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts
index b2a03a7623bfe7..f51c15f882a4e9 100644
--- a/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts
+++ b/src/core/server/saved_objects/migrations/kibana/kibana_migrator.ts
@@ -106,11 +106,12 @@ export class KibanaMigrator {
         documentMigrator: this.documentMigrator,
         index,
         log: this.log,
-        mappingProperties: indexMap[index],
+        mappingProperties: indexMap[index].typeMappings,
         pollInterval: config.get('migrations.pollInterval'),
         scrollDuration: config.get('migrations.scrollDuration'),
         serializer: this.serializer,
         obsoleteIndexTemplatePattern: 'kibana_index_template*',
+        convertToAliasScript: indexMap[index].script,
       });
     });
 
diff --git a/src/core/server/saved_objects/schema/schema.ts b/src/core/server/saved_objects/schema/schema.ts
index 6756feeb15a0f6..1f098d0b6e21d6 100644
--- a/src/core/server/saved_objects/schema/schema.ts
+++ b/src/core/server/saved_objects/schema/schema.ts
@@ -20,6 +20,7 @@ interface SavedObjectsSchemaTypeDefinition {
   isNamespaceAgnostic: boolean;
   hidden?: boolean;
   indexPattern?: string;
+  convertToAliasScript?: string;
 }
 
 export interface SavedObjectsSchemaDefinition {

From f9435af9f6a1669de6f2c951806f0efefb05156f Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 16 Jul 2019 23:47:43 -0400
Subject: [PATCH 37/48] Convert task manager to use saved objects

---
 src/core/server/index.ts                      |   3 +
 src/core/server/saved_objects/index.ts        |   2 +
 .../alerting/server/alerts_client.test.ts     |  19 +-
 .../server/maps_telemetry/telemetry_task.js   |  30 +-
 .../server/lib/get_next_midnight.ts           |   2 +-
 .../oss_telemetry/server/lib/tasks/index.ts   |  28 +-
 .../legacy/plugins/task_manager/constants.ts  |   6 +-
 x-pack/legacy/plugins/task_manager/index.js   |  31 +-
 .../legacy/plugins/task_manager/mappings.json |  42 +++
 .../legacy/plugins/task_manager/migrations.js |  14 +
 x-pack/legacy/plugins/task_manager/task.ts    |   9 +-
 .../plugins/task_manager/task_manager.test.ts |  68 ++--
 .../plugins/task_manager/task_manager.ts      |  66 ++--
 .../plugins/task_manager/task_poller.test.ts  |  54 ---
 .../plugins/task_manager/task_poller.ts       |   8 -
 .../plugins/task_manager/task_runner.ts       |   3 +-
 .../plugins/task_manager/task_store.test.ts   | 313 ++++++++---------
 .../legacy/plugins/task_manager/task_store.ts | 323 +++---------------
 18 files changed, 410 insertions(+), 611 deletions(-)
 create mode 100644 x-pack/legacy/plugins/task_manager/mappings.json
 create mode 100644 x-pack/legacy/plugins/task_manager/migrations.js

diff --git a/src/core/server/index.ts b/src/core/server/index.ts
index 787478d5b3c3f8..6c2e9cff79b00c 100644
--- a/src/core/server/index.ts
+++ b/src/core/server/index.ts
@@ -87,6 +87,7 @@ export {
 } from './plugins';
 
 export {
+  RawSavedObjectDoc,
   SavedObject,
   SavedObjectAttributes,
   SavedObjectReference,
@@ -103,6 +104,8 @@ export {
   SavedObjectsFindOptions,
   SavedObjectsFindResponse,
   SavedObjectsMigrationVersion,
+  SavedObjectsSchema,
+  SavedObjectsSerializer,
   SavedObjectsService,
   SavedObjectsUpdateOptions,
   SavedObjectsUpdateResponse,
diff --git a/src/core/server/saved_objects/index.ts b/src/core/server/saved_objects/index.ts
index e6e9e2d2660004..865f8d9750570b 100644
--- a/src/core/server/saved_objects/index.ts
+++ b/src/core/server/saved_objects/index.ts
@@ -22,3 +22,5 @@ export * from './service';
 export { SavedObjectsSchema } from './schema';
 
 export { SavedObjectsManagement } from './management';
+
+export { SavedObjectsSerializer, RawDoc as RawSavedObjectDoc } from './serialization';
diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
index 7a3e7f82121ffd..5e452e98b73a08 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
@@ -94,8 +94,6 @@ describe('create()', () => {
     taskManager.schedule.mockResolvedValueOnce({
       id: 'task-123',
       taskType: 'alerting:123',
-      sequenceNumber: 1,
-      primaryTerm: 1,
       scheduledAt: new Date(),
       attempts: 1,
       status: 'idle',
@@ -440,8 +438,6 @@ describe('enable()', () => {
     });
     taskManager.schedule.mockResolvedValueOnce({
       id: 'task-123',
-      sequenceNumber: 1,
-      primaryTerm: 1,
       scheduledAt: new Date(),
       attempts: 0,
       status: 'idle',
@@ -449,6 +445,8 @@ describe('enable()', () => {
       state: {},
       params: {},
       taskType: '',
+      startedAt: null,
+      retryAt: null,
     });
 
     await alertsClient.enable({ id: '1' });
@@ -740,19 +738,8 @@ describe('delete()', () => {
     savedObjectsClient.delete.mockResolvedValueOnce({
       success: true,
     });
-    taskManager.remove.mockResolvedValueOnce({
-      index: '.task_manager',
-      id: 'task-123',
-      sequenceNumber: 1,
-      primaryTerm: 1,
-      result: '',
-    });
     const result = await alertsClient.delete({ id: '1' });
-    expect(result).toMatchInlineSnapshot(`
-                              Object {
-                                "success": true,
-                              }
-                    `);
+    expect(result).toBeUndefined();
     expect(savedObjectsClient.delete).toHaveBeenCalledTimes(1);
     expect(savedObjectsClient.delete.mock.calls[0]).toMatchInlineSnapshot(`
                               Array [
diff --git a/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.js b/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.js
index 7fbbe8ef77ff50..1f1a9e369a9524 100644
--- a/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.js
+++ b/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.js
@@ -13,16 +13,24 @@ export const TASK_ID = `Maps-${TELEMETRY_TASK_TYPE}`;
 export function scheduleTask(server, taskManager) {
   const { kbnServer } = server.plugins.xpack_main.status.plugin;
 
-  kbnServer.afterPluginsInit(async () => {
-    try {
-      await taskManager.schedule({
-        id: TASK_ID,
-        taskType: TELEMETRY_TASK_TYPE,
-        state: { stats: {}, runs: 0 },
-      });
-    }catch(e) {
-      server.log(['warning', 'maps'], `Error scheduling telemetry task, received ${e.message}`);
-    }
+  kbnServer.afterPluginsInit(() => {
+    // The code block below can't await directly within "afterPluginsInit"
+    // callback due to circular dependency. The server isn't "ready" until
+    // this code block finishes. Migrations wait for server to be ready before
+    // executing. Saved objects repository waits for migrations to finish before
+    // finishing the request. To avoid this, we'll await within a separate
+    // function block.
+    (async () => {
+      try {
+        await taskManager.schedule({
+          id: TASK_ID,
+          taskType: TELEMETRY_TASK_TYPE,
+          state: { stats: {}, runs: 0 },
+        });
+      }catch(e) {
+        server.log(['warning', 'maps'], `Error scheduling telemetry task, received ${e.message}`);
+      }
+    })();
   });
 }
 
@@ -73,5 +81,5 @@ export function getNextMidnight() {
   const nextMidnight = new Date();
   nextMidnight.setHours(0, 0, 0, 0);
   nextMidnight.setDate(nextMidnight.getDate() + 1);
-  return nextMidnight.toISOString();
+  return nextMidnight;
 }
diff --git a/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.ts b/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.ts
index c286af1854b6ab..a5ee8d572343ce 100644
--- a/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.ts
+++ b/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.ts
@@ -8,5 +8,5 @@ export function getNextMidnight() {
   const nextMidnight = new Date();
   nextMidnight.setHours(0, 0, 0, 0);
   nextMidnight.setDate(nextMidnight.getDate() + 1);
-  return nextMidnight.toISOString();
+  return nextMidnight;
 }
diff --git a/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/index.ts b/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/index.ts
index 8c0771aba11500..bc00e39a2886c4 100644
--- a/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/index.ts
+++ b/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/index.ts
@@ -29,15 +29,23 @@ export function scheduleTasks(server: HapiServer) {
   const { taskManager } = server;
   const { kbnServer } = server.plugins.xpack_main.status.plugin;
 
-  kbnServer.afterPluginsInit(async () => {
-    try {
-      await taskManager.schedule({
-        id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}`,
-        taskType: VIS_TELEMETRY_TASK,
-        state: { stats: {}, runs: 0 },
-      });
-    } catch (e) {
-      server.log(['warning', 'telemetry'], `Error scheduling task, received ${e.message}`);
-    }
+  kbnServer.afterPluginsInit(() => {
+    // The code block below can't await directly within "afterPluginsInit"
+    // callback due to circular dependency. The server isn't "ready" until
+    // this code block finishes. Migrations wait for server to be ready before
+    // executing. Saved objects repository waits for migrations to finish before
+    // finishing the request. To avoid this, we'll await within a separate
+    // function block.
+    (async () => {
+      try {
+        await taskManager.schedule({
+          id: `${PLUGIN_ID}-${VIS_TELEMETRY_TASK}`,
+          taskType: VIS_TELEMETRY_TASK,
+          state: { stats: {}, runs: 0 },
+        });
+      } catch (e) {
+        server.log(['warning', 'telemetry'], `Error scheduling task, received ${e.message}`);
+      }
+    })();
   });
 }
diff --git a/x-pack/legacy/plugins/task_manager/constants.ts b/x-pack/legacy/plugins/task_manager/constants.ts
index a43ac1893b685c..1b0d0b001071c3 100644
--- a/x-pack/legacy/plugins/task_manager/constants.ts
+++ b/x-pack/legacy/plugins/task_manager/constants.ts
@@ -4,8 +4,4 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import xPackage from '../../../package.json';
-import { getTemplateVersion } from './lib/get_template_version';
-
-export const TASK_MANAGER_API_VERSION = 1;
-export const TASK_MANAGER_TEMPLATE_VERSION = getTemplateVersion(xPackage.version);
+export const TASK_MANAGER_INDEX = '.kibana_task_manager';
diff --git a/x-pack/legacy/plugins/task_manager/index.js b/x-pack/legacy/plugins/task_manager/index.js
index a6f2b8adee81b5..7d166b9f3afbc6 100644
--- a/x-pack/legacy/plugins/task_manager/index.js
+++ b/x-pack/legacy/plugins/task_manager/index.js
@@ -4,7 +4,11 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { SavedObjectsSerializer, SavedObjectsSchema } from '../../../../src/core/server';
 import { TaskManager } from './task_manager';
+import mappings from './mappings.json';
+import migrations from './migrations';
+import { TASK_MANAGER_INDEX } from './constants';
 
 export function taskManager(kibana) {
   return new kibana.Plugin({
@@ -37,8 +41,33 @@ export function taskManager(kibana) {
     },
     init(server) {
       const config = server.config();
-      const taskManager = new TaskManager(this.kbnServer, server, config);
+      const schema = new SavedObjectsSchema(this.kbnServer.uiExports.savedObjectSchemas);
+      const serializer = new SavedObjectsSerializer(schema);
+      const { callWithInternalUser } = server.plugins.elasticsearch.getCluster('admin');
+      const savedObjectsRepository = server.savedObjects.getSavedObjectsRepository(
+        callWithInternalUser,
+        ['task']
+      );
+
+      const taskManager = new TaskManager({
+        kbnServer: this.kbnServer,
+        config,
+        savedObjectsRepository,
+        serializer,
+      });
       server.decorate('server', 'taskManager', taskManager);
     },
+    uiExports: {
+      mappings,
+      migrations,
+      savedObjectSchemas: {
+        task: {
+          hidden: true,
+          isNamespaceAgnostic: true,
+          indexPattern: TASK_MANAGER_INDEX,
+          convertToAliasScript: `ctx._id = ctx._source.type + ':' + ctx._id`,
+        },
+      },
+    },
   });
 }
diff --git a/x-pack/legacy/plugins/task_manager/mappings.json b/x-pack/legacy/plugins/task_manager/mappings.json
new file mode 100644
index 00000000000000..6638a75d825466
--- /dev/null
+++ b/x-pack/legacy/plugins/task_manager/mappings.json
@@ -0,0 +1,42 @@
+{
+  "task": {
+    "properties": {
+      "taskType": {
+        "type": "keyword"
+      },
+      "scheduledAt": {
+        "type": "date"
+      },
+      "runAt": {
+        "type": "date"
+      },
+      "startedAt": {
+        "type": "date"
+      },
+      "retryAt": {
+        "type": "date"
+      },
+      "interval": {
+        "type": "text"
+      },
+      "attempts": {
+        "type": "integer"
+      },
+      "status": {
+        "type": "keyword"
+      },
+      "params": {
+        "type": "text"
+      },
+      "state": {
+        "type": "text"
+      },
+      "user": {
+        "type": "keyword"
+      },
+      "scope": {
+        "type": "keyword"
+      }
+    }
+  }
+}
diff --git a/x-pack/legacy/plugins/task_manager/migrations.js b/x-pack/legacy/plugins/task_manager/migrations.js
new file mode 100644
index 00000000000000..a9027a4dbd54d5
--- /dev/null
+++ b/x-pack/legacy/plugins/task_manager/migrations.js
@@ -0,0 +1,14 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default {
+  task: {
+    '7.4.0': (doc) => {
+      doc.updated_at = new Date().toISOString();
+      return doc;
+    }
+  }
+};
diff --git a/x-pack/legacy/plugins/task_manager/task.ts b/x-pack/legacy/plugins/task_manager/task.ts
index a15be155649a47..dfb8ce6c5f9f95 100644
--- a/x-pack/legacy/plugins/task_manager/task.ts
+++ b/x-pack/legacy/plugins/task_manager/task.ts
@@ -241,14 +241,9 @@ export interface ConcreteTaskInstance extends TaskInstance {
   id: string;
 
   /**
-   * The sequence number from the Elaticsearch document.
+   * The saved object version from the Elaticsearch document.
    */
-  sequenceNumber: number;
-
-  /**
-   * The primary term from the Elaticsearch document.
-   */
-  primaryTerm: number;
+  version?: string;
 
   /**
    * The date and time that this task was originally scheduled. This is used
diff --git a/x-pack/legacy/plugins/task_manager/task_manager.test.ts b/x-pack/legacy/plugins/task_manager/task_manager.test.ts
index 58f3c6f50bc33e..f2d4c6bf9bb90b 100644
--- a/x-pack/legacy/plugins/task_manager/task_manager.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_manager.test.ts
@@ -7,6 +7,11 @@
 import _ from 'lodash';
 import sinon from 'sinon';
 import { TaskManager } from './task_manager';
+import { SavedObjectsClientMock } from 'src/core/server/mocks';
+import { SavedObjectsSerializer, SavedObjectsSchema } from 'src/core/server';
+
+const savedObjectsClient = SavedObjectsClientMock.create();
+const serializer = new SavedObjectsSerializer(new SavedObjectsSchema());
 
 describe('TaskManager', () => {
   let clock: sinon.SinonFakeTimers;
@@ -28,7 +33,12 @@ describe('TaskManager', () => {
 
   test('disallows schedule before init', async () => {
     const { opts } = testOpts();
-    const client = new TaskManager(opts.kbnServer, opts.server, opts.config);
+    const client = new TaskManager({
+      kbnServer: opts.kbnServer,
+      config: opts.config,
+      savedObjectsRepository: savedObjectsClient,
+      serializer,
+    });
     const task = {
       taskType: 'foo',
       params: {},
@@ -39,19 +49,34 @@ describe('TaskManager', () => {
 
   test('disallows fetch before init', async () => {
     const { opts } = testOpts();
-    const client = new TaskManager(opts.kbnServer, opts.server, opts.config);
+    const client = new TaskManager({
+      kbnServer: opts.kbnServer,
+      config: opts.config,
+      savedObjectsRepository: savedObjectsClient,
+      serializer,
+    });
     await expect(client.fetch({})).rejects.toThrow(/^NotInitialized: .*/i);
   });
 
   test('disallows remove before init', async () => {
     const { opts } = testOpts();
-    const client = new TaskManager(opts.kbnServer, opts.server, opts.config);
+    const client = new TaskManager({
+      kbnServer: opts.kbnServer,
+      config: opts.config,
+      savedObjectsRepository: savedObjectsClient,
+      serializer,
+    });
     await expect(client.remove('23')).rejects.toThrow(/^NotInitialized: .*/i);
   });
 
   test('allows middleware registration before init', () => {
     const { opts } = testOpts();
-    const client = new TaskManager(opts.kbnServer, opts.server, opts.config);
+    const client = new TaskManager({
+      kbnServer: opts.kbnServer,
+      config: opts.config,
+      savedObjectsRepository: savedObjectsClient,
+      serializer,
+    });
     const middleware = {
       beforeSave: async (saveOpts: any) => saveOpts,
       beforeRun: async (runOpts: any) => runOpts,
@@ -61,7 +86,12 @@ describe('TaskManager', () => {
 
   test('disallows middleware registration after init', async () => {
     const { $test, opts } = testOpts();
-    const client = new TaskManager(opts.kbnServer, opts.server, opts.config);
+    const client = new TaskManager({
+      kbnServer: opts.kbnServer,
+      config: opts.config,
+      savedObjectsRepository: savedObjectsClient,
+      serializer,
+    });
     const middleware = {
       beforeSave: async (saveOpts: any) => saveOpts,
       beforeRun: async (runOpts: any) => runOpts,
@@ -94,20 +124,20 @@ describe('TaskManager', () => {
         afterPluginsInit(callback: any) {
           $test.afterPluginsInit = callback;
         },
-      },
-      server: {
-        log: sinon.spy(),
-        decorate(...args: any[]) {
-          _.set(opts, args.slice(0, -1), _.last(args));
-        },
-        plugins: {
-          elasticsearch: {
-            getCluster() {
-              return { callWithInternalUser: callCluster };
-            },
-            status: {
-              on(eventName: string, callback: () => any) {
-                $test.events[eventName] = callback;
+        server: {
+          log: sinon.spy(),
+          decorate(...args: any[]) {
+            _.set(opts, args.slice(0, -1), _.last(args));
+          },
+          plugins: {
+            elasticsearch: {
+              getCluster() {
+                return { callWithInternalUser: callCluster };
+              },
+              status: {
+                on(eventName: string, callback: () => any) {
+                  $test.events[eventName] = callback;
+                },
               },
             },
           },
diff --git a/x-pack/legacy/plugins/task_manager/task_manager.ts b/x-pack/legacy/plugins/task_manager/task_manager.ts
index 1ea45b434d75b0..4a74422e565cd7 100644
--- a/x-pack/legacy/plugins/task_manager/task_manager.ts
+++ b/x-pack/legacy/plugins/task_manager/task_manager.ts
@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { SavedObjectsClientContract, SavedObjectsSerializer } from 'src/core/server';
 import { fillPool } from './lib/fill_pool';
 import { Logger, TaskManagerLogger } from './lib/logger';
 import { addMiddlewareToChain, BeforeSaveMiddlewareParams, Middleware } from './lib/middleware';
@@ -13,7 +14,14 @@ import { SanitizedTaskDefinition, TaskDefinition, TaskDictionary } from './task'
 import { TaskPoller } from './task_poller';
 import { TaskPool } from './task_pool';
 import { TaskManagerRunner } from './task_runner';
-import { FetchOpts, FetchResult, RemoveResult, TaskStore } from './task_store';
+import { FetchOpts, FetchResult, TaskStore } from './task_store';
+
+export interface TaskManagerOpts {
+  kbnServer: any;
+  config: any;
+  savedObjectsRepository: SavedObjectsClientContract;
+  serializer: SavedObjectsSerializer;
+}
 
 /*
  * The TaskManager is the public interface into the task manager system. This glues together
@@ -46,9 +54,10 @@ export class TaskManager {
    * enabling the task manipulation methods, and beginning the background polling
    * mechanism.
    */
-  public constructor(kbnServer: any, server: any, config: any) {
-    this.maxWorkers = config.get('xpack.task_manager.max_workers');
-    this.overrideNumWorkers = config.get('xpack.task_manager.override_num_workers');
+  public constructor(opts: TaskManagerOpts) {
+    const { server } = opts.kbnServer;
+    this.maxWorkers = opts.config.get('xpack.task_manager.max_workers');
+    this.overrideNumWorkers = opts.config.get('xpack.task_manager.override_num_workers');
     this.definitions = {};
 
     const logger = new TaskManagerLogger((...args: any[]) => server.log(...args));
@@ -56,12 +65,11 @@ export class TaskManager {
     /* Kibana UUID needs to be pulled live (not cached), as it takes a long time
      * to initialize, and can change after startup */
     const store = new TaskStore({
+      serializer: opts.serializer,
+      savedObjectsRepository: opts.savedObjectsRepository,
       callCluster: server.plugins.elasticsearch.getCluster('admin').callWithInternalUser,
-      index: config.get('xpack.task_manager.index'),
-      maxAttempts: config.get('xpack.task_manager.max_attempts'),
+      maxAttempts: opts.config.get('xpack.task_manager.max_attempts'),
       definitions: this.definitions,
-      logger,
-      getKibanaUuid: () => config.get('server.uuid'),
     });
     const pool = new TaskPool({
       logger,
@@ -70,7 +78,7 @@ export class TaskManager {
     const createRunner = (instance: ConcreteTaskInstance) =>
       new TaskManagerRunner({
         logger,
-        kbnServer,
+        kbnServer: opts.kbnServer,
         instance,
         store,
         definitions: this.definitions,
@@ -78,8 +86,7 @@ export class TaskManager {
       });
     const poller = new TaskPoller({
       logger,
-      pollInterval: config.get('xpack.task_manager.poll_interval'),
-      store,
+      pollInterval: opts.config.get('xpack.task_manager.poll_interval'),
       work(): Promise<void> {
         return fillPool(pool.run, store.fetchAvailableTasks, createRunner);
       },
@@ -89,24 +96,25 @@ export class TaskManager {
     this.store = store;
     this.poller = poller;
 
-    kbnServer.afterPluginsInit(async () => {
-      const startPoller = () => {
-        return poller
-          .start()
-          .then(() => {
-            this.isInitialized = true;
-          })
-          .catch((err: Error) => {
-            // FIXME: check the type of error to make sure it's actually an ES error
-            logger.warning(`PollError ${err.message}`);
-
-            // rety again to initialize store and poller, using the timing of
-            // task_manager's configurable poll interval
-            const retryInterval = config.get('xpack.task_manager.poll_interval');
-            setTimeout(() => startPoller(), retryInterval);
-          });
+    opts.kbnServer.afterPluginsInit(() => {
+      // By this point, the plugins had their chance to register task definitions
+      // and we're good to start doing CRUD actions
+      this.isInitialized = true;
+      const startPoller = async () => {
+        await server.kibanaMigrator.awaitMigration();
+        try {
+          await poller.start();
+        } catch (err) {
+          // FIXME: check the type of error to make sure it's actually an ES error
+          logger.warning(`PollError ${err.message}`);
+
+          // rety again to initialize store and poller, using the timing of
+          // task_manager's configurable poll interval
+          const retryInterval = opts.config.get('xpack.task_manager.poll_interval');
+          setTimeout(() => startPoller(), retryInterval);
+        }
       };
-      return startPoller();
+      startPoller();
     });
   }
 
@@ -179,7 +187,7 @@ export class TaskManager {
    * @param {string} id
    * @returns {Promise<RemoveResult>}
    */
-  public async remove(id: string): Promise<RemoveResult> {
+  public async remove(id: string): Promise<void> {
     this.assertInitialized('Tasks cannot be removed before task manager is initialized!');
     return this.store.remove(id);
   }
diff --git a/x-pack/legacy/plugins/task_manager/task_poller.test.ts b/x-pack/legacy/plugins/task_manager/task_poller.test.ts
index f61d569cead4b7..478c1a4dc1b17c 100644
--- a/x-pack/legacy/plugins/task_manager/task_poller.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_poller.test.ts
@@ -7,46 +7,12 @@
 import _ from 'lodash';
 import sinon from 'sinon';
 import { TaskPoller } from './task_poller';
-import { TaskStore } from './task_store';
 import { mockLogger, resolvable, sleep } from './test_utils';
-import { TaskDictionary, SanitizedTaskDefinition } from './task';
-
-let store: TaskStore;
-
-const taskDefinitions: TaskDictionary<SanitizedTaskDefinition> = {
-  report: {
-    type: 'report',
-    title: '',
-    numWorkers: 1,
-    createTaskRunner: jest.fn(),
-  },
-  dernstraight: {
-    type: 'dernstraight',
-    title: '',
-    numWorkers: 1,
-    createTaskRunner: jest.fn(),
-  },
-  yawn: {
-    type: 'yawn',
-    title: '',
-    numWorkers: 1,
-    createTaskRunner: jest.fn(),
-  },
-};
 
 describe('TaskPoller', () => {
   beforeEach(() => {
     const callCluster = sinon.stub();
     callCluster.withArgs('indices.getTemplate').returns(Promise.resolve({ tasky: {} }));
-    const getKibanaUuid = sinon.stub().returns('kibana-123-uuid-test');
-    store = new TaskStore({
-      callCluster,
-      getKibanaUuid,
-      logger: mockLogger(),
-      index: 'tasky',
-      maxAttempts: 2,
-      definitions: taskDefinitions,
-    });
   });
 
   describe('interval tests', () => {
@@ -66,7 +32,6 @@ describe('TaskPoller', () => {
         return Promise.resolve();
       });
       const poller = new TaskPoller({
-        store,
         pollInterval,
         work,
         logger: mockLogger(),
@@ -89,7 +54,6 @@ describe('TaskPoller', () => {
     const logger = mockLogger();
     const doneWorking = resolvable();
     const poller = new TaskPoller({
-      store,
       logger,
       pollInterval: 1,
       work: async () => {
@@ -120,7 +84,6 @@ describe('TaskPoller', () => {
     });
 
     const poller = new TaskPoller({
-      store,
       logger: mockLogger(),
       pollInterval: 1,
       work,
@@ -139,7 +102,6 @@ describe('TaskPoller', () => {
       await doneWorking;
     });
     const poller = new TaskPoller({
-      store,
       pollInterval: 1,
       logger: mockLogger(),
       work,
@@ -165,7 +127,6 @@ describe('TaskPoller', () => {
       doneWorking.resolve();
     });
     const poller = new TaskPoller({
-      store,
       pollInterval: 1,
       logger: mockLogger(),
       work,
@@ -176,19 +137,4 @@ describe('TaskPoller', () => {
 
     sinon.assert.calledOnce(work);
   });
-
-  test('start method passes through error from store.init', async () => {
-    store.init = () => {
-      throw new Error('test error');
-    };
-
-    const poller = new TaskPoller({
-      store,
-      pollInterval: 1,
-      logger: mockLogger(),
-      work: sinon.stub(),
-    });
-
-    await expect(poller.start()).rejects.toMatchInlineSnapshot(`[Error: test error]`);
-  });
 });
diff --git a/x-pack/legacy/plugins/task_manager/task_poller.ts b/x-pack/legacy/plugins/task_manager/task_poller.ts
index e77d9c97fba0df..739f2d35a36750 100644
--- a/x-pack/legacy/plugins/task_manager/task_poller.ts
+++ b/x-pack/legacy/plugins/task_manager/task_poller.ts
@@ -9,14 +9,12 @@
  */
 
 import { Logger } from './lib/logger';
-import { TaskStore } from './task_store';
 
 type WorkFn = () => Promise<void>;
 
 interface Opts {
   pollInterval: number;
   logger: Logger;
-  store: TaskStore;
   work: WorkFn;
 }
 
@@ -30,7 +28,6 @@ export class TaskPoller {
   private timeout: any;
   private pollInterval: number;
   private logger: Logger;
-  private store: TaskStore;
   private work: WorkFn;
 
   /**
@@ -44,7 +41,6 @@ export class TaskPoller {
   constructor(opts: Opts) {
     this.pollInterval = opts.pollInterval;
     this.logger = opts.logger;
-    this.store = opts.store;
     this.work = opts.work;
   }
 
@@ -56,10 +52,6 @@ export class TaskPoller {
       return;
     }
 
-    if (!this.store.isInitialized) {
-      await this.store.init();
-    }
-
     this.isStarted = true;
 
     const poll = async () => {
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 4387a54e65f489..50286d531e1c11 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -24,7 +24,6 @@ import {
   TaskDictionary,
   validateRunResult,
 } from './task';
-import { RemoveResult } from './task_store';
 
 export interface TaskRunner {
   numWorkers: number;
@@ -38,7 +37,7 @@ export interface TaskRunner {
 interface Updatable {
   readonly maxAttempts: number;
   update(doc: ConcreteTaskInstance): Promise<ConcreteTaskInstance>;
-  remove(id: string): Promise<RemoveResult>;
+  remove(id: string): Promise<void>;
 }
 
 interface Opts {
diff --git a/x-pack/legacy/plugins/task_manager/task_store.test.ts b/x-pack/legacy/plugins/task_manager/task_store.test.ts
index 302125e4ac90e5..1d50ec79a150b6 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.test.ts
@@ -6,15 +6,11 @@
 
 import _ from 'lodash';
 import sinon from 'sinon';
-import {
-  TASK_MANAGER_API_VERSION as API_VERSION,
-  TASK_MANAGER_TEMPLATE_VERSION as TEMPLATE_VERSION,
-} from './constants';
 import { TaskDictionary, SanitizedTaskDefinition, TaskInstance, TaskStatus } from './task';
 import { FetchOpts, TaskStore } from './task_store';
 import { mockLogger } from './test_utils';
-
-const getKibanaUuid = sinon.stub().returns('kibana-uuid-123-test');
+import { SavedObjectsClientMock } from 'src/core/server/mocks';
+import { SavedObjectsSerializer, SavedObjectsSchema, SavedObjectAttributes } from 'src/core/server';
 
 const taskDefinitions: TaskDictionary<SanitizedTaskDefinition> = {
   report: {
@@ -37,94 +33,46 @@ const taskDefinitions: TaskDictionary<SanitizedTaskDefinition> = {
   },
 };
 
-describe('TaskStore', () => {
-  describe('init', () => {
-    test('creates the task manager index', async () => {
-      const callCluster = sinon.stub();
-      callCluster.withArgs('indices.getTemplate').returns(Promise.resolve({ tasky: {} }));
-      const store = new TaskStore({
-        callCluster,
-        getKibanaUuid,
-        logger: mockLogger(),
-        index: 'tasky',
-        maxAttempts: 2,
-        definitions: taskDefinitions,
-      });
-
-      await store.init();
-
-      sinon.assert.calledTwice(callCluster); // store.init calls twice: once to check for existing template, once to put the template (if needed)
-
-      sinon.assert.calledWithMatch(callCluster, 'indices.putTemplate', {
-        body: {
-          index_patterns: ['tasky'],
-          settings: {
-            number_of_shards: 1,
-            auto_expand_replicas: '0-1',
-          },
-        },
-        name: 'tasky',
-      });
-    });
-
-    test('logs a warning if newer index template exists', async () => {
-      const callCluster = sinon.stub();
-      callCluster
-        .withArgs('indices.getTemplate')
-        .returns(Promise.resolve({ tasky: { version: Infinity } }));
-
-      const logger = {
-        info: sinon.spy(),
-        debug: sinon.spy(),
-        warning: sinon.spy(),
-        error: sinon.spy(),
-      };
+const savedObjectsClient = SavedObjectsClientMock.create();
+const serializer = new SavedObjectsSerializer(new SavedObjectsSchema());
 
-      const store = new TaskStore({
-        callCluster,
-        getKibanaUuid,
-        logger,
-        index: 'tasky',
-        maxAttempts: 2,
-        definitions: taskDefinitions,
-      });
+beforeEach(() => jest.resetAllMocks());
 
-      await store.init();
-      const loggingCall = logger.warning.getCall(0);
-      expect(loggingCall.args[0]).toBe(
-        `This Kibana instance defines an older template version (${TEMPLATE_VERSION}) than is currently in Elasticsearch (Infinity). ` +
-          `Because of the potential for non-backwards compatible changes, this Kibana instance will only be able to claim scheduled tasks with ` +
-          `"kibana.apiVersion" <= ${API_VERSION} in the task metadata.`
-      );
-      expect(logger.warning.calledOnce).toBe(true);
-    });
-  });
+const mockedDate = new Date('2019-02-12T21:01:22.479Z');
+(global as any).Date = class Date {
+  constructor() {
+    return mockedDate;
+  }
+  static now() {
+    return mockedDate.getTime();
+  }
+};
 
+describe('TaskStore', () => {
   describe('schedule', () => {
     async function testSchedule(task: TaskInstance) {
-      const callCluster = sinon.stub();
-      callCluster.withArgs('index').returns(
-        Promise.resolve({
-          _id: 'testid',
-          _seq_no: 3344,
-          _primary_term: 3344,
+      const callCluster = jest.fn();
+      savedObjectsClient.create.mockImplementation(
+        async (type: string, attributes: SavedObjectAttributes) => ({
+          id: 'testid',
+          type,
+          attributes,
+          references: [],
+          version: '123',
         })
       );
-      callCluster.withArgs('indices.getTemplate').returns(Promise.resolve({ tasky: {} }));
       const store = new TaskStore({
+        serializer,
         callCluster,
-        getKibanaUuid,
-        logger: mockLogger(),
-        index: 'tasky',
         maxAttempts: 2,
         definitions: taskDefinitions,
+        savedObjectsRepository: savedObjectsClient,
       });
-      await store.init();
       const result = await store.schedule(task);
 
-      sinon.assert.calledThrice(callCluster);
+      expect(savedObjectsClient.create).toHaveBeenCalledTimes(1);
 
-      return { result, callCluster, arg: callCluster.args[2][1] };
+      return result;
     }
 
     test('serializes the params and state', async () => {
@@ -133,18 +81,42 @@ describe('TaskStore', () => {
         state: { foo: 'bar' },
         taskType: 'report',
       };
-      const { callCluster, arg } = await testSchedule(task);
-
-      sinon.assert.calledWith(callCluster, 'index');
+      const result = await testSchedule(task);
 
-      expect(arg).toMatchObject({
-        index: 'tasky',
-        body: {
-          task: {
-            params: JSON.stringify(task.params),
-            state: JSON.stringify(task.state),
-          },
+      expect(savedObjectsClient.create).toHaveBeenCalledWith(
+        'task',
+        {
+          attempts: 0,
+          interval: undefined,
+          params: '{"hello":"world"}',
+          retryAt: null,
+          runAt: '2019-02-12T21:01:22.479Z',
+          scheduledAt: '2019-02-12T21:01:22.479Z',
+          scope: undefined,
+          startedAt: null,
+          state: '{"foo":"bar"}',
+          status: 'idle',
+          taskType: 'report',
+          user: undefined,
         },
+        {}
+      );
+
+      expect(result).toEqual({
+        id: 'testid',
+        attempts: 0,
+        interval: undefined,
+        params: { hello: 'world' },
+        retryAt: null,
+        runAt: mockedDate,
+        scheduledAt: mockedDate,
+        scope: undefined,
+        startedAt: null,
+        state: { foo: 'bar' },
+        status: 'idle',
+        taskType: 'report',
+        user: undefined,
+        version: '123',
       });
     });
 
@@ -154,26 +126,27 @@ describe('TaskStore', () => {
         state: { foo: 'bar' },
         taskType: 'report',
       };
-      const { result } = await testSchedule(task);
+      const result = await testSchedule(task);
 
       expect(result).toMatchObject({
         ...task,
-        sequenceNumber: 3344,
-        primaryTerm: 3344,
         id: 'testid',
       });
     });
 
     test('sets runAt to now if not specified', async () => {
-      const now = Date.now();
-      const { arg } = await testSchedule({ taskType: 'dernstraight', params: {}, state: {} });
-      expect(arg.body.task.runAt.getTime()).toBeGreaterThanOrEqual(now);
+      await testSchedule({ taskType: 'dernstraight', params: {}, state: {} });
+      expect(savedObjectsClient.create).toHaveBeenCalledTimes(1);
+      const attributes = savedObjectsClient.create.mock.calls[0][1];
+      expect(new Date(attributes.runAt as string).getTime()).toEqual(mockedDate.getTime());
     });
 
     test('ensures params and state are not null', async () => {
-      const { arg } = await testSchedule({ taskType: 'yawn' } as any);
-      expect(arg.body.task.params).toEqual('{}');
-      expect(arg.body.task.state).toEqual('{}');
+      await testSchedule({ taskType: 'yawn' } as any);
+      expect(savedObjectsClient.create).toHaveBeenCalledTimes(1);
+      const attributes = savedObjectsClient.create.mock.calls[0][1];
+      expect(attributes.params).toEqual('{}');
+      expect(attributes.state).toEqual('{}');
     });
 
     test('errors if the task type is unknown', async () => {
@@ -187,12 +160,11 @@ describe('TaskStore', () => {
     async function testFetch(opts?: FetchOpts, hits: any[] = []) {
       const callCluster = sinon.spy(async () => ({ hits: { hits } }));
       const store = new TaskStore({
+        serializer,
         callCluster,
-        getKibanaUuid,
-        logger: mockLogger(),
-        index: 'tasky',
         maxAttempts: 2,
         definitions: taskDefinitions,
+        savedObjectsRepository: savedObjectsClient,
       });
 
       const result = await store.fetch(opts);
@@ -209,7 +181,7 @@ describe('TaskStore', () => {
     test('empty call filters by type, sorts by runAt and id', async () => {
       const { args } = await testFetch();
       expect(args).toMatchObject({
-        index: 'tasky',
+        index: '.kibana_task_manager',
         body: {
           sort: [{ 'task.runAt': 'asc' }, { _id: 'desc' }],
           query: { term: { type: 'task' } },
@@ -322,13 +294,14 @@ describe('TaskStore', () => {
             interval: undefined,
             params: { hello: 'world' },
             runAt,
+            scheduledAt: mockedDate,
             scope: ['reporting'],
             state: { baby: 'Henhen' },
             status: 'idle',
             taskType: 'foo',
             user: 'jimbo',
-            sequenceNumber: undefined,
-            primaryTerm: undefined,
+            retryAt: undefined,
+            startedAt: undefined,
           },
           {
             attempts: 2,
@@ -336,13 +309,14 @@ describe('TaskStore', () => {
             interval: '5m',
             params: { shazm: 1 },
             runAt,
+            scheduledAt: mockedDate,
             scope: ['reporting', 'ceo'],
             state: { henry: 'The 8th' },
             status: 'running',
             taskType: 'bar',
             user: 'dabo',
-            sequenceNumber: undefined,
-            primaryTerm: undefined,
+            retryAt: undefined,
+            startedAt: undefined,
           },
         ],
         searchAfter: ['b', 2],
@@ -357,8 +331,8 @@ describe('TaskStore', () => {
         callCluster,
         logger: mockLogger(),
         definitions: taskDefinitions,
-        index: 'tasky',
         maxAttempts: 2,
+        serializer,
         ...opts,
       });
 
@@ -376,12 +350,11 @@ describe('TaskStore', () => {
     test('it returns normally with no tasks when the index does not exist.', async () => {
       const callCluster = sinon.spy(async () => ({ hits: { hits: [] } }));
       const store = new TaskStore({
+        serializer,
         callCluster,
-        getKibanaUuid,
-        logger: mockLogger(),
         definitions: taskDefinitions,
-        index: 'tasky',
         maxAttempts: 2,
+        savedObjectsRepository: savedObjectsClient,
       });
 
       const result = await store.fetchAvailableTasks();
@@ -395,10 +368,8 @@ describe('TaskStore', () => {
     test('it filters tasks by supported types, maxAttempts, and runAt', async () => {
       const maxAttempts = _.random(2, 43);
       const customMaxAttempts = _.random(44, 100);
-      const index = `index_${_.random(1, 234)}`;
       const { args } = await testFetchAvailableTasks({
         opts: {
-          index,
           maxAttempts,
           definitions: {
             foo: {
@@ -426,7 +397,6 @@ describe('TaskStore', () => {
                 {
                   bool: {
                     must: [
-                      { range: { 'kibana.apiVersion': { lte: 1 } } },
                       {
                         bool: {
                           should: [
@@ -503,7 +473,6 @@ describe('TaskStore', () => {
           },
           seq_no_primary_term: true,
         },
-        index,
       });
     });
 
@@ -527,6 +496,8 @@ describe('TaskStore', () => {
                 scope: ['reporting'],
               },
             },
+            _seq_no: 1,
+            _primary_term: 2,
             sort: ['a', 1],
           },
           {
@@ -545,6 +516,8 @@ describe('TaskStore', () => {
                 scope: ['reporting', 'ceo'],
               },
             },
+            _seq_no: 3,
+            _primary_term: 4,
             sort: ['b', 2],
           },
         ],
@@ -561,8 +534,6 @@ describe('TaskStore', () => {
           status: 'idle',
           taskType: 'foo',
           user: 'jimbo',
-          sequenceNumber: undefined,
-          primaryTerm: undefined,
         },
         {
           attempts: 2,
@@ -575,8 +546,6 @@ describe('TaskStore', () => {
           status: 'running',
           taskType: 'bar',
           user: 'dabo',
-          sequenceNumber: undefined,
-          primaryTerm: undefined,
         },
       ]);
     });
@@ -584,62 +553,70 @@ describe('TaskStore', () => {
 
   describe('update', () => {
     test('refreshes the index, handles versioning', async () => {
-      const runAt = new Date();
       const task = {
-        runAt,
-        scheduledAt: runAt,
+        runAt: mockedDate,
+        scheduledAt: mockedDate,
         startedAt: null,
         retryAt: null,
         id: 'task:324242',
         params: { hello: 'world' },
         state: { foo: 'bar' },
         taskType: 'report',
-        sequenceNumber: 2,
-        primaryTerm: 2,
         attempts: 3,
         status: 'idle' as TaskStatus,
+        version: '123',
       };
 
-      const callCluster = sinon.spy(async () => ({
-        _seq_no: task.sequenceNumber + 1,
-        _primary_term: task.primaryTerm + 1,
-      }));
+      savedObjectsClient.update.mockImplementation(
+        async (type: string, id: string, attributes: SavedObjectAttributes) => {
+          return {
+            id,
+            type,
+            attributes,
+            references: [],
+            version: '123',
+          };
+        }
+      );
 
       const store = new TaskStore({
-        callCluster,
-        getKibanaUuid,
-        logger: mockLogger(),
-        index: 'tasky',
+        serializer,
+        callCluster: jest.fn(),
         maxAttempts: 2,
         definitions: taskDefinitions,
+        savedObjectsRepository: savedObjectsClient,
       });
 
       const result = await store.update(task);
 
-      sinon.assert.calledOnce(callCluster);
-      sinon.assert.calledWith(callCluster, 'update');
-
-      expect(callCluster.args[0][1]).toMatchObject({
-        id: task.id,
-        index: 'tasky',
-        if_seq_no: 2,
-        if_primary_term: 2,
-        refresh: true,
-        body: {
-          doc: {
-            task: {
-              ..._.omit(task, ['id', 'sequenceNumber', 'primaryTerm']),
-              params: JSON.stringify(task.params),
-              state: JSON.stringify(task.state),
-            },
-          },
+      expect(savedObjectsClient.update).toHaveBeenCalledWith(
+        'task',
+        task.id,
+        {
+          attempts: task.attempts,
+          interval: undefined,
+          params: JSON.stringify(task.params),
+          retryAt: null,
+          runAt: task.runAt.toISOString(),
+          scheduledAt: mockedDate.toISOString(),
+          scope: undefined,
+          startedAt: null,
+          state: JSON.stringify(task.state),
+          status: task.status,
+          taskType: task.taskType,
+          user: undefined,
         },
-      });
+        { version: '123' }
+      );
 
       expect(result).toEqual({
         ...task,
-        sequenceNumber: 3,
-        primaryTerm: 3,
+        interval: undefined,
+        retryAt: null,
+        scope: undefined,
+        startedAt: null,
+        user: undefined,
+        version: '123',
       });
     });
   });
@@ -647,41 +624,17 @@ describe('TaskStore', () => {
   describe('remove', () => {
     test('removes the task with the specified id', async () => {
       const id = `id-${_.random(1, 20)}`;
-      const callCluster = sinon.spy(() =>
-        Promise.resolve({
-          _index: 'myindex',
-          _id: id,
-          _seq_no: 32,
-          _primary_term: 32,
-          result: 'deleted',
-        })
-      );
+      const callCluster = jest.fn();
       const store = new TaskStore({
+        serializer,
         callCluster,
-        getKibanaUuid,
-        logger: mockLogger(),
-        index: 'myindex',
         maxAttempts: 2,
         definitions: taskDefinitions,
+        savedObjectsRepository: savedObjectsClient,
       });
       const result = await store.remove(id);
-
-      sinon.assert.calledOnce(callCluster);
-      sinon.assert.calledWith(callCluster, 'delete');
-
-      expect(result).toEqual({
-        id,
-        index: 'myindex',
-        sequenceNumber: 32,
-        primaryTerm: 32,
-        result: 'deleted',
-      });
-
-      expect(callCluster.args[0][1]).toMatchObject({
-        id,
-        index: 'myindex',
-        refresh: true,
-      });
+      expect(result).toBeUndefined();
+      expect(savedObjectsClient.delete).toHaveBeenCalledWith('task', id);
     });
   });
 });
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index af0956a7f6836b..e4df0021127dc7 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -8,27 +8,29 @@
  * This module contains helpers for managing the task manager storage layer.
  */
 
+import { omit } from 'lodash';
 import {
-  TASK_MANAGER_API_VERSION as API_VERSION,
-  TASK_MANAGER_TEMPLATE_VERSION as TEMPLATE_VERSION,
-} from './constants';
-import { Logger } from './lib/logger';
+  SavedObjectsClientContract,
+  SavedObject,
+  SavedObjectAttributes,
+  SavedObjectsSerializer,
+  RawSavedObjectDoc,
+} from 'src/core/server';
 import {
   ConcreteTaskInstance,
   ElasticJs,
   SanitizedTaskDefinition,
   TaskDictionary,
   TaskInstance,
-  TaskStatus,
 } from './task';
+import { TASK_MANAGER_INDEX } from './constants';
 
 export interface StoreOpts {
   callCluster: ElasticJs;
-  getKibanaUuid: () => string;
-  index: string;
   maxAttempts: number;
   definitions: TaskDictionary<SanitizedTaskDefinition>;
-  logger: Logger;
+  savedObjectsRepository: SavedObjectsClientContract;
+  serializer: SavedObjectsSerializer;
 }
 
 export interface FetchOpts {
@@ -42,192 +44,42 @@ export interface FetchResult {
   docs: ConcreteTaskInstance[];
 }
 
-export interface RemoveResult {
-  index: string;
-  id: string;
-  sequenceNumber: number;
-  primaryTerm: number;
-  result: string;
-}
-
-// Internal, the raw document, as stored in the Kibana index.
-export interface RawTaskDoc {
-  _id: string;
-  _index: string;
-  _seq_no: number;
-  _primary_term: number;
-  _source: {
-    type: string;
-    kibana: {
-      uuid: string;
-      version: number;
-      apiVersion: number;
-    };
-    task: {
-      taskType: string;
-      scheduledAt: Date;
-      runAt: Date;
-      startedAt: Date | null;
-      retryAt: Date | null;
-      interval?: string;
-      attempts: number;
-      status: TaskStatus;
-      params: string;
-      state: string;
-      user?: string;
-      scope?: string[];
-    };
-  };
-}
-
 /**
  * Wraps an elasticsearch connection and provides a task manager-specific
  * interface into the index.
  */
 export class TaskStore {
   public readonly maxAttempts: number;
-  public getKibanaUuid: () => string;
-  public readonly index: string;
   private callCluster: ElasticJs;
   private definitions: TaskDictionary<SanitizedTaskDefinition>;
-  private _isInitialized = false; // eslint-disable-line @typescript-eslint/camelcase
-  private logger: Logger;
+  private savedObjectsRepository: SavedObjectsClientContract;
+  private serializer: SavedObjectsSerializer;
 
   /**
    * Constructs a new TaskStore.
    * @param {StoreOpts} opts
    * @prop {CallCluster} callCluster - The elastic search connection
-   * @prop {string} index - The name of the task manager index
    * @prop {number} maxAttempts - The maximum number of attempts before a task will be abandoned
    * @prop {TaskDefinition} definition - The definition of the task being run
-   * @prop {Logger} logger - The task manager logger.
+   * @prop {serializer} - The saved object serializer
+   * @prop {savedObjectsRepository} - An instance to the saved objects repository
    */
   constructor(opts: StoreOpts) {
     this.callCluster = opts.callCluster;
-    this.index = opts.index;
     this.maxAttempts = opts.maxAttempts;
     this.definitions = opts.definitions;
-    this.logger = opts.logger;
-    this.getKibanaUuid = opts.getKibanaUuid;
+    this.serializer = opts.serializer;
+    this.savedObjectsRepository = opts.savedObjectsRepository;
 
     this.fetchAvailableTasks = this.fetchAvailableTasks.bind(this);
   }
 
-  /**
-   * Initializes the store, ensuring the task manager index template is created
-   * and the version is up to date.
-   */
-  public async init() {
-    if (this._isInitialized) {
-      throw new Error('TaskStore has already been initialized!');
-    }
-
-    let existingVersion = -Infinity;
-    const templateName = this.index;
-
-    try {
-      // check if template exists
-      const templateCheck = await this.callCluster('indices.getTemplate', {
-        name: templateName,
-        filter_path: '*.version',
-      });
-      // extract the existing version
-      const template = templateCheck[templateName] || {};
-      existingVersion = template.version || 0;
-    } catch (err) {
-      if (err.statusCode !== 404) {
-        throw err; // ignore not found
-      }
-    }
-
-    if (existingVersion > TEMPLATE_VERSION) {
-      // Do not trample a newer version template
-      this.logger.warning(
-        `This Kibana instance defines an older template version (${TEMPLATE_VERSION}) than is currently in Elasticsearch (${existingVersion}). ` +
-          `Because of the potential for non-backwards compatible changes, this Kibana instance will only be able to claim scheduled tasks with ` +
-          `"kibana.apiVersion" <= ${API_VERSION} in the task metadata.`
-      );
-      return;
-    } else if (existingVersion === TEMPLATE_VERSION) {
-      // The latest template is already saved, so just log a debug line.
-      this.logger.debug(
-        `Not installing ${this.index} index template: version ${TEMPLATE_VERSION} already exists.`
-      );
-      return;
-    }
-
-    // Activate template creation / update
-    if (existingVersion > 0) {
-      this.logger.info(
-        `Upgrading ${this.index} index template. Old version: ${existingVersion}, New version: ${TEMPLATE_VERSION}.`
-      );
-    } else {
-      this.logger.info(`Installing ${this.index} index template version: ${TEMPLATE_VERSION}.`);
-    }
-
-    const templateResult = await this.callCluster('indices.putTemplate', {
-      name: templateName,
-      body: {
-        index_patterns: [this.index],
-        mappings: {
-          dynamic: false,
-          properties: {
-            type: { type: 'keyword' },
-            task: {
-              properties: {
-                taskType: { type: 'keyword' },
-                scheduledAt: { type: 'date' },
-                runAt: { type: 'date' },
-                startedAt: { type: 'date' },
-                retryAt: { type: 'date' },
-                interval: { type: 'text' },
-                attempts: { type: 'integer' },
-                status: { type: 'keyword' },
-                params: { type: 'text' },
-                state: { type: 'text' },
-                user: { type: 'keyword' },
-                scope: { type: 'keyword' },
-              },
-            },
-            kibana: {
-              properties: {
-                apiVersion: { type: 'integer' }, // 1, 2, 3, etc
-                uuid: { type: 'keyword' }, //
-                version: { type: 'integer' }, // 7000099, etc
-              },
-            },
-          },
-        },
-        settings: {
-          number_of_shards: 1,
-          auto_expand_replicas: '0-1',
-        },
-        version: TEMPLATE_VERSION,
-      },
-    });
-
-    this._isInitialized = true;
-    this.logger.info(
-      `Installed ${this.index} index template: version ${TEMPLATE_VERSION} (API version ${API_VERSION})`
-    );
-
-    return templateResult;
-  }
-
-  get isInitialized() {
-    return this._isInitialized;
-  }
-
   /**
    * Schedules a task.
    *
    * @param task - The task being scheduled.
    */
   public async schedule(taskInstance: TaskInstance): Promise<ConcreteTaskInstance> {
-    if (!this._isInitialized) {
-      await this.init();
-    }
-
     if (!this.definitions[taskInstance.taskType]) {
       throw new Error(
         `Unsupported task type "${taskInstance.taskType}". Supported types are ${Object.keys(
@@ -236,28 +88,13 @@ export class TaskStore {
       );
     }
 
-    const { id, ...body } = rawSource(taskInstance, this);
-    const result = await this.callCluster('index', {
-      id,
-      body,
-      index: this.index,
-      refresh: true,
-    });
+    const savedObject = await this.savedObjectsRepository.create(
+      'task',
+      taskInstanceToAttributes(taskInstance),
+      { id: taskInstance.id }
+    );
 
-    const { task } = body;
-    return {
-      ...taskInstance,
-      id: result._id,
-      sequenceNumber: result._seq_no,
-      primaryTerm: result._primary_term,
-      attempts: 0,
-      status: task.status,
-      scheduledAt: task.scheduledAt,
-      startedAt: null,
-      retryAt: null,
-      runAt: task.runAt,
-      state: taskInstance.state || {},
-    };
+    return savedObjectToConcreteTaskInstance(savedObject);
   }
 
   /**
@@ -290,7 +127,6 @@ export class TaskStore {
       query: {
         bool: {
           must: [
-            { range: { 'kibana.apiVersion': { lte: API_VERSION } } },
             // Either a task with idle status and runAt <= now or
             // status running with a retryAt <= now.
             {
@@ -365,26 +201,14 @@ export class TaskStore {
    * @returns {Promise<TaskDoc>}
    */
   public async update(doc: ConcreteTaskInstance): Promise<ConcreteTaskInstance> {
-    const rawDoc = taskDocToRaw(doc, this);
-
-    const result = await this.callCluster('update', {
-      body: {
-        doc: rawDoc._source,
-      },
-      id: doc.id,
-      index: this.index,
-      if_seq_no: doc.sequenceNumber,
-      if_primary_term: doc.primaryTerm,
-      // The refresh is important so that if we immediately look for work,
-      // we don't pick up this task.
-      refresh: true,
-    });
+    const updatedSavedObject = await this.savedObjectsRepository.update(
+      'task',
+      doc.id,
+      taskInstanceToAttributes(doc),
+      { version: doc.version }
+    );
 
-    return {
-      ...doc,
-      sequenceNumber: result._seq_no,
-      primaryTerm: result._primary_term,
-    };
+    return savedObjectToConcreteTaskInstance(updatedSavedObject);
   }
 
   /**
@@ -393,22 +217,8 @@ export class TaskStore {
    * @param {string} id
    * @returns {Promise<void>}
    */
-  public async remove(id: string): Promise<RemoveResult> {
-    const result = await this.callCluster('delete', {
-      id,
-      index: this.index,
-      // The refresh is important so that if we immediately look for work,
-      // we don't pick up this task.
-      refresh: true,
-    });
-
-    return {
-      index: result._index,
-      id: result._id,
-      sequenceNumber: result._seq_no,
-      primaryTerm: result._primary_term,
-      result: result.result,
-    };
+  public async remove(id: string): Promise<void> {
+    await this.savedObjectsRepository.delete('task', id);
   }
 
   private async search(opts: any = {}): Promise<FetchResult> {
@@ -419,7 +229,7 @@ export class TaskStore {
       : queryOnlyTasks;
 
     const result = await this.callCluster('search', {
-      index: this.index,
+      index: TASK_MANAGER_INDEX,
       ignoreUnavailable: true,
       body: {
         ...opts,
@@ -430,7 +240,10 @@ export class TaskStore {
     const rawDocs = result.hits.hits;
 
     return {
-      docs: (rawDocs as RawTaskDoc[]).map(rawToTaskDoc),
+      docs: (rawDocs as RawSavedObjectDoc[])
+        .map(doc => this.serializer.rawToSavedObject(doc))
+        .map(doc => omit(doc, 'namespace') as SavedObject)
+        .map(savedObjectToConcreteTaskInstance),
       searchAfter: (rawDocs.length && rawDocs[rawDocs.length - 1].sort) || [],
     };
   }
@@ -450,64 +263,38 @@ function paginatableSort(sort: any[] = []) {
   return [...sort, sortById];
 }
 
-function rawSource(doc: TaskInstance, store: TaskStore) {
-  const { id, ...taskFields } = doc;
-  const source = {
-    ...taskFields,
+function taskInstanceToAttributes(doc: TaskInstance): SavedObjectAttributes {
+  return {
+    ...omit(doc, 'id', 'version'),
     params: JSON.stringify(doc.params || {}),
     state: JSON.stringify(doc.state || {}),
     attempts: (doc as ConcreteTaskInstance).attempts || 0,
-    scheduledAt: doc.scheduledAt || new Date(),
-    startedAt: doc.startedAt || null,
-    retryAt: doc.retryAt || null,
-    runAt: doc.runAt || new Date(),
+    scheduledAt: (doc.scheduledAt || new Date()).toISOString(),
+    startedAt: (doc.startedAt && doc.startedAt.toISOString()) || null,
+    retryAt: (doc.retryAt && doc.retryAt.toISOString()) || null,
+    runAt: (doc.runAt || new Date()).toISOString(),
     status: (doc as ConcreteTaskInstance).status || 'idle',
   };
-
-  delete (source as any).id;
-  delete (source as any).sequenceNumber;
-  delete (source as any).primaryTerm;
-  delete (source as any).type;
-
-  return {
-    id,
-    type: 'task',
-    task: source,
-    kibana: {
-      uuid: store.getKibanaUuid(), // needs to be pulled live
-      version: TEMPLATE_VERSION,
-      apiVersion: API_VERSION,
-    },
-  };
-}
-
-function taskDocToRaw(doc: ConcreteTaskInstance, store: TaskStore): RawTaskDoc {
-  const { type, task, kibana } = rawSource(doc, store);
-
-  return {
-    _id: doc.id,
-    _index: store.index,
-    _source: { type, task, kibana },
-    _seq_no: doc.sequenceNumber,
-    _primary_term: doc.primaryTerm,
-  };
 }
 
-function rawToTaskDoc(doc: RawTaskDoc): ConcreteTaskInstance {
+function savedObjectToConcreteTaskInstance(savedObject: SavedObject): ConcreteTaskInstance {
   return {
-    ...doc._source.task,
-    id: doc._id,
-    sequenceNumber: doc._seq_no,
-    primaryTerm: doc._primary_term,
-    params: parseJSONField(doc._source.task.params, 'params', doc),
-    state: parseJSONField(doc._source.task.state, 'state', doc),
+    ...savedObject.attributes,
+    id: savedObject.id,
+    version: savedObject.version,
+    scheduledAt: new Date(savedObject.attributes.scheduledAt),
+    runAt: new Date(savedObject.attributes.runAt),
+    startedAt: savedObject.attributes.startedAt && new Date(savedObject.attributes.startedAt),
+    retryAt: savedObject.attributes.retryAt && new Date(savedObject.attributes.retryAt),
+    state: parseJSONField(savedObject.attributes.state, 'state', savedObject.id),
+    params: parseJSONField(savedObject.attributes.params, 'params', savedObject.id),
   };
 }
 
-function parseJSONField(json: string, fieldName: string, doc: RawTaskDoc) {
+function parseJSONField(json: string, fieldName: string, id: string) {
   try {
     return json ? JSON.parse(json) : {};
   } catch (error) {
-    throw new Error(`Task "${doc._id}"'s ${fieldName} field has invalid JSON: ${json}`);
+    throw new Error(`Task "${id}"'s ${fieldName} field has invalid JSON: ${json}`);
   }
 }

From 6ee7dbc1fa2864ad5c7a169649dbf39b8c07e83a Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 16 Jul 2019 23:51:35 -0400
Subject: [PATCH 38/48] Fix broken test

---
 x-pack/legacy/plugins/task_manager/task_manager.test.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/x-pack/legacy/plugins/task_manager/task_manager.test.ts b/x-pack/legacy/plugins/task_manager/task_manager.test.ts
index f2d4c6bf9bb90b..77a8139b1d1c50 100644
--- a/x-pack/legacy/plugins/task_manager/task_manager.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_manager.test.ts
@@ -129,6 +129,9 @@ describe('TaskManager', () => {
           decorate(...args: any[]) {
             _.set(opts, args.slice(0, -1), _.last(args));
           },
+          kibanaMigrator: {
+            awaitMigration: jest.fn(),
+          },
           plugins: {
             elasticsearch: {
               getCluster() {

From 64cb12d3d03d1e595183e0a4e1c3f66ee1e00f23 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 17 Jul 2019 08:05:18 -0400
Subject: [PATCH 39/48] Fix broken tests pt1

---
 x-pack/legacy/plugins/alerting/server/alerts_client.test.ts     | 2 +-
 .../plugins/maps/server/maps_telemetry/telemetry_task.test.js   | 2 +-
 .../plugins/oss_telemetry/server/lib/get_next_midnight.test.ts  | 2 +-
 .../server/lib/tasks/visualizations/task_runner.test.ts         | 2 +-
 x-pack/test/api_integration/apis/alerting/create.ts             | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
index 5e452e98b73a08..9dbe38155d689c 100644
--- a/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
+++ b/x-pack/legacy/plugins/alerting/server/alerts_client.test.ts
@@ -739,7 +739,7 @@ describe('delete()', () => {
       success: true,
     });
     const result = await alertsClient.delete({ id: '1' });
-    expect(result).toBeUndefined();
+    expect(result).toEqual({ success: true });
     expect(savedObjectsClient.delete).toHaveBeenCalledTimes(1);
     expect(savedObjectsClient.delete.mock.calls[0]).toMatchInlineSnapshot(`
                               Array [
diff --git a/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.test.js b/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.test.js
index 58d53d35260c70..1e149ed62b09f0 100644
--- a/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.test.js
+++ b/x-pack/legacy/plugins/maps/server/maps_telemetry/telemetry_task.test.js
@@ -25,7 +25,7 @@ describe('telemetryTaskRunner', () => {
       moment()
         .add(1, 'days')
         .startOf('day')
-        .toISOString();
+        .toDate();
 
     const getRunner = telemetryTaskRunner();
     const runResult = await getRunner(
diff --git a/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts b/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts
index d8df006a55b7f1..c20e4a0b4be543 100644
--- a/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts
+++ b/x-pack/legacy/plugins/oss_telemetry/server/lib/get_next_midnight.test.ts
@@ -12,7 +12,7 @@ describe('getNextMidnight', () => {
     const nextMidnightMoment = moment()
       .add(1, 'days')
       .startOf('day')
-      .toISOString();
+      .toDate();
 
     expect(getNextMidnight()).toEqual(nextMidnightMoment);
   });
diff --git a/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts b/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts
index 879846aeed1ae2..f7e3ae7091c736 100644
--- a/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts
+++ b/x-pack/legacy/plugins/oss_telemetry/server/lib/tasks/visualizations/task_runner.test.ts
@@ -43,7 +43,7 @@ describe('visualizationsTaskRunner', () => {
       moment()
         .add(1, 'days')
         .startOf('day')
-        .toISOString();
+        .toDate();
 
     const runner = visualizationsTaskRunner(mockTaskInstance, { server: mockKbnServer });
     const result = await runner();
diff --git a/x-pack/test/api_integration/apis/alerting/create.ts b/x-pack/test/api_integration/apis/alerting/create.ts
index ebe033708019fb..c345be264e310e 100644
--- a/x-pack/test/api_integration/apis/alerting/create.ts
+++ b/x-pack/test/api_integration/apis/alerting/create.ts
@@ -33,7 +33,7 @@ export default function createAlertTests({ getService }: KibanaFunctionalTestDef
 
     async function getScheduledTask(id: string) {
       return await es.get({
-        id,
+        id: `task:${id}`,
         index: '.kibana_task_manager',
       });
     }

From 66cb2bcdb204eee97832ba55b7121a80a4f886ae Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 17 Jul 2019 08:45:20 -0400
Subject: [PATCH 40/48] Remove index from task manager config schema

---
 x-pack/legacy/plugins/task_manager/index.js | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/index.js b/x-pack/legacy/plugins/task_manager/index.js
index 7d166b9f3afbc6..8365e03ae594cb 100644
--- a/x-pack/legacy/plugins/task_manager/index.js
+++ b/x-pack/legacy/plugins/task_manager/index.js
@@ -26,9 +26,6 @@ export function taskManager(kibana) {
           .description('How often, in milliseconds, the task manager will look for more work.')
           .min(1000)
           .default(3000),
-        index: Joi.string()
-          .description('The name of the index used to store task information.')
-          .default('.kibana_task_manager'),
         max_workers: Joi.number()
           .description('The maximum number of tasks that this Kibana instance will run simultaneously.')
           .min(1) // disable the task manager rather than trying to specify it with 0 workers

From af07c3c7b69668eea3c07bbea87e8d74b9f3d14a Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 17 Jul 2019 08:49:26 -0400
Subject: [PATCH 41/48] Accept platform changes

---
 .../core/server/kibana-plugin-server.md       | 179 +++++++++---------
 ...ana-plugin-server.rawsavedobjectdoc._id.md |  11 ++
 ...-server.rawsavedobjectdoc._primary_term.md |  11 ++
 ...plugin-server.rawsavedobjectdoc._seq_no.md |  11 ++
 ...plugin-server.rawsavedobjectdoc._source.md |  11 ++
 ...a-plugin-server.rawsavedobjectdoc._type.md |  11 ++
 .../kibana-plugin-server.rawsavedobjectdoc.md |  24 +++
 ...server.savedobjectsschema.(constructor).md |  20 ++
 ...rver.savedobjectsschema.getindexfortype.md |  22 +++
 ...-server.savedobjectsschema.ishiddentype.md |  22 +++
 ....savedobjectsschema.isnamespaceagnostic.md |  22 +++
 ...kibana-plugin-server.savedobjectsschema.md |  26 +++
 ...er.savedobjectsserializer.(constructor).md |  20 ++
 ...er.savedobjectsserializer.generaterawid.md |  26 +++
 ...savedobjectsserializer.israwsavedobject.md |  24 +++
 ...na-plugin-server.savedobjectsserializer.md |  27 +++
 ...savedobjectsserializer.rawtosavedobject.md |  24 +++
 ...savedobjectsserializer.savedobjecttoraw.md |  24 +++
 src/core/server/server.api.md                 |  42 ++++
 19 files changed, 469 insertions(+), 88 deletions(-)
 create mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._id.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._type.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsschema.(constructor).md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsschema.getindexfortype.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsschema.ishiddentype.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsschema.isnamespaceagnostic.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsschema.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsserializer.(constructor).md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsserializer.generaterawid.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsserializer.israwsavedobject.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsserializer.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsserializer.rawtosavedobject.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsserializer.savedobjecttoraw.md

diff --git a/docs/development/core/server/kibana-plugin-server.md b/docs/development/core/server/kibana-plugin-server.md
index ab79f2b3829094..d8d53421d68748 100644
--- a/docs/development/core/server/kibana-plugin-server.md
+++ b/docs/development/core/server/kibana-plugin-server.md
@@ -1,88 +1,91 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md)
-
-## kibana-plugin-server package
-
-The Kibana Core APIs for server-side plugins.
-
-A plugin's `server/index` file must contain a named import, `plugin`<!-- -->, that implements [PluginInitializer](./kibana-plugin-server.plugininitializer.md) which returns an object that implements [Plugin](./kibana-plugin-server.plugin.md)<!-- -->.
-
-The plugin integrates with the core system via lifecycle events: `setup`<!-- -->, `start`<!-- -->, and `stop`<!-- -->. In each lifecycle method, the plugin will receive the corresponding core services available (either [CoreSetup](./kibana-plugin-server.coresetup.md) or [CoreStart](./kibana-plugin-server.corestart.md)<!-- -->) and any interfaces returned by dependency plugins' lifecycle method. Anything returned by the plugin's lifecycle method will be exposed to downstream dependencies when their corresponding lifecycle methods are invoked.
-
-## Classes
-
-|  Class | Description |
-|  --- | --- |
-|  [ClusterClient](./kibana-plugin-server.clusterclient.md) | Represents an Elasticsearch cluster API client and allows to call API on behalf of the internal Kibana user and the actual user that is derived from the request headers (via <code>asScoped(...)</code>). |
-|  [ElasticsearchErrorHelpers](./kibana-plugin-server.elasticsearcherrorhelpers.md) | Helpers for working with errors returned from the Elasticsearch service.Since the internal data of errors are subject to change, consumers of the Elasticsearch service should always use these helpers to classify errors instead of checking error internals such as <code>body.error.header[WWW-Authenticate]</code> |
-|  [KibanaRequest](./kibana-plugin-server.kibanarequest.md) | Kibana specific abstraction for an incoming request. |
-|  [Router](./kibana-plugin-server.router.md) |  |
-|  [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |  |
-|  [ScopedClusterClient](./kibana-plugin-server.scopedclusterclient.md) | Serves the same purpose as "normal" <code>ClusterClient</code> but exposes additional <code>callAsCurrentUser</code> method that doesn't use credentials of the Kibana internal user (as <code>callAsInternalUser</code> does) to request Elasticsearch API, but rather passes HTTP headers extracted from the current user request to the API |
-
-## Interfaces
-
-|  Interface | Description |
-|  --- | --- |
-|  [AuthResultData](./kibana-plugin-server.authresultdata.md) | Result of an incoming request authentication. |
-|  [AuthToolkit](./kibana-plugin-server.authtoolkit.md) | A tool set defining an outcome of Auth interceptor for incoming request. |
-|  [CallAPIOptions](./kibana-plugin-server.callapioptions.md) | The set of options that defines how API call should be made and result be processed. |
-|  [CoreSetup](./kibana-plugin-server.coresetup.md) | Context passed to the plugins <code>setup</code> method. |
-|  [CoreStart](./kibana-plugin-server.corestart.md) | Context passed to the plugins <code>start</code> method. |
-|  [DiscoveredPlugin](./kibana-plugin-server.discoveredplugin.md) | Small container object used to expose information about discovered plugins that may or may not have been started. |
-|  [ElasticsearchError](./kibana-plugin-server.elasticsearcherror.md) |  |
-|  [ElasticsearchServiceSetup](./kibana-plugin-server.elasticsearchservicesetup.md) |  |
-|  [FakeRequest](./kibana-plugin-server.fakerequest.md) | Fake request object created manually by Kibana plugins. |
-|  [HttpServiceSetup](./kibana-plugin-server.httpservicesetup.md) |  |
-|  [HttpServiceStart](./kibana-plugin-server.httpservicestart.md) |  |
-|  [InternalCoreStart](./kibana-plugin-server.internalcorestart.md) |  |
-|  [KibanaRequestRoute](./kibana-plugin-server.kibanarequestroute.md) | Request specific route information exposed to a handler. |
-|  [Logger](./kibana-plugin-server.logger.md) | Logger exposes all the necessary methods to log any type of information and this is the interface used by the logging consumers including plugins. |
-|  [LoggerFactory](./kibana-plugin-server.loggerfactory.md) | The single purpose of <code>LoggerFactory</code> interface is to define a way to retrieve a context-based logger instance. |
-|  [LogMeta](./kibana-plugin-server.logmeta.md) | Contextual metadata |
-|  [OnPostAuthToolkit](./kibana-plugin-server.onpostauthtoolkit.md) | A tool set defining an outcome of OnPostAuth interceptor for incoming request. |
-|  [OnPreAuthToolkit](./kibana-plugin-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |
-|  [Plugin](./kibana-plugin-server.plugin.md) | The interface that should be returned by a <code>PluginInitializer</code>. |
-|  [PluginInitializerContext](./kibana-plugin-server.plugininitializercontext.md) | Context that's available to plugins during initialization stage. |
-|  [PluginsServiceSetup](./kibana-plugin-server.pluginsservicesetup.md) |  |
-|  [PluginsServiceStart](./kibana-plugin-server.pluginsservicestart.md) |  |
-|  [RouteConfigOptions](./kibana-plugin-server.routeconfigoptions.md) | Route specific configuration. |
-|  [SavedObject](./kibana-plugin-server.savedobject.md) |  |
-|  [SavedObjectAttributes](./kibana-plugin-server.savedobjectattributes.md) |  |
-|  [SavedObjectReference](./kibana-plugin-server.savedobjectreference.md) | A reference to another saved object. |
-|  [SavedObjectsBaseOptions](./kibana-plugin-server.savedobjectsbaseoptions.md) |  |
-|  [SavedObjectsBulkCreateObject](./kibana-plugin-server.savedobjectsbulkcreateobject.md) |  |
-|  [SavedObjectsBulkGetObject](./kibana-plugin-server.savedobjectsbulkgetobject.md) |  |
-|  [SavedObjectsBulkResponse](./kibana-plugin-server.savedobjectsbulkresponse.md) |  |
-|  [SavedObjectsClientWrapperOptions](./kibana-plugin-server.savedobjectsclientwrapperoptions.md) |  |
-|  [SavedObjectsCreateOptions](./kibana-plugin-server.savedobjectscreateoptions.md) |  |
-|  [SavedObjectsFindOptions](./kibana-plugin-server.savedobjectsfindoptions.md) |  |
-|  [SavedObjectsFindResponse](./kibana-plugin-server.savedobjectsfindresponse.md) |  |
-|  [SavedObjectsMigrationVersion](./kibana-plugin-server.savedobjectsmigrationversion.md) | A dictionary of saved object type -<!-- -->&gt; version used to determine what migrations need to be applied to a saved object. |
-|  [SavedObjectsService](./kibana-plugin-server.savedobjectsservice.md) |  |
-|  [SavedObjectsUpdateOptions](./kibana-plugin-server.savedobjectsupdateoptions.md) |  |
-|  [SavedObjectsUpdateResponse](./kibana-plugin-server.savedobjectsupdateresponse.md) |  |
-|  [SessionStorage](./kibana-plugin-server.sessionstorage.md) | Provides an interface to store and retrieve data across requests. |
-|  [SessionStorageFactory](./kibana-plugin-server.sessionstoragefactory.md) | SessionStorage factory to bind one to an incoming request |
-
-## Type Aliases
-
-|  Type Alias | Description |
-|  --- | --- |
-|  [APICaller](./kibana-plugin-server.apicaller.md) |  |
-|  [AuthenticationHandler](./kibana-plugin-server.authenticationhandler.md) |  |
-|  [AuthHeaders](./kibana-plugin-server.authheaders.md) | Auth Headers map |
-|  [ElasticsearchClientConfig](./kibana-plugin-server.elasticsearchclientconfig.md) |  |
-|  [GetAuthHeaders](./kibana-plugin-server.getauthheaders.md) | Get headers to authenticate a user against Elasticsearch. |
-|  [Headers](./kibana-plugin-server.headers.md) |  |
-|  [LegacyRequest](./kibana-plugin-server.legacyrequest.md) | Support Legacy platform request for the period of migration. |
-|  [OnPostAuthHandler](./kibana-plugin-server.onpostauthhandler.md) |  |
-|  [OnPreAuthHandler](./kibana-plugin-server.onpreauthhandler.md) |  |
-|  [PluginInitializer](./kibana-plugin-server.plugininitializer.md) | The <code>plugin</code> export at the root of a plugin's <code>server</code> directory should conform to this interface. |
-|  [PluginName](./kibana-plugin-server.pluginname.md) | Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays that use it as a key or value more obvious. |
-|  [RecursiveReadonly](./kibana-plugin-server.recursivereadonly.md) |  |
-|  [RouteMethod](./kibana-plugin-server.routemethod.md) | The set of common HTTP methods supported by Kibana routing. |
-|  [SavedObjectsClientContract](./kibana-plugin-server.savedobjectsclientcontract.md) | \#\# SavedObjectsClient errors<!-- -->Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:<!-- -->1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md)<!-- -->Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the <code>isXYZError()</code> helpers exposed at <code>SavedObjectsErrorHelpers</code> should be used to understand and manage error responses from the <code>SavedObjectsClient</code>.<!-- -->Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for <code>error.body.error.type</code> or doing substring checks on <code>error.body.error.reason</code>, just use the helpers to understand the meaning of the error:<!-- -->\`\`\`<!-- -->js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }<!-- -->if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }<!-- -->// always rethrow the error unless you handle it throw error; \`\`\`<!-- -->\#\#\# 404s from missing index<!-- -->From the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.<!-- -->At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.<!-- -->From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.<!-- -->\#\#\# 503s from missing index<!-- -->Unlike all other methods, create requests are supposed to succeed even when the Kibana index does not exist because it will be automatically created by elasticsearch. When that is not the case it is because Elasticsearch's <code>action.auto_create_index</code> setting prevents it from being created automatically so we throw a special 503 with the intention of informing the user that their Elasticsearch settings need to be updated.<!-- -->See [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |
-|  [SavedObjectsClientWrapperFactory](./kibana-plugin-server.savedobjectsclientwrapperfactory.md) |  |
-
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md)
+
+## kibana-plugin-server package
+
+The Kibana Core APIs for server-side plugins.
+
+A plugin's `server/index` file must contain a named import, `plugin`<!-- -->, that implements [PluginInitializer](./kibana-plugin-server.plugininitializer.md) which returns an object that implements [Plugin](./kibana-plugin-server.plugin.md)<!-- -->.
+
+The plugin integrates with the core system via lifecycle events: `setup`<!-- -->, `start`<!-- -->, and `stop`<!-- -->. In each lifecycle method, the plugin will receive the corresponding core services available (either [CoreSetup](./kibana-plugin-server.coresetup.md) or [CoreStart](./kibana-plugin-server.corestart.md)<!-- -->) and any interfaces returned by dependency plugins' lifecycle method. Anything returned by the plugin's lifecycle method will be exposed to downstream dependencies when their corresponding lifecycle methods are invoked.
+
+## Classes
+
+|  Class | Description |
+|  --- | --- |
+|  [ClusterClient](./kibana-plugin-server.clusterclient.md) | Represents an Elasticsearch cluster API client and allows to call API on behalf of the internal Kibana user and the actual user that is derived from the request headers (via <code>asScoped(...)</code>). |
+|  [ElasticsearchErrorHelpers](./kibana-plugin-server.elasticsearcherrorhelpers.md) | Helpers for working with errors returned from the Elasticsearch service.Since the internal data of errors are subject to change, consumers of the Elasticsearch service should always use these helpers to classify errors instead of checking error internals such as <code>body.error.header[WWW-Authenticate]</code> |
+|  [KibanaRequest](./kibana-plugin-server.kibanarequest.md) | Kibana specific abstraction for an incoming request. |
+|  [Router](./kibana-plugin-server.router.md) |  |
+|  [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |  |
+|  [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md) |  |
+|  [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) |  |
+|  [ScopedClusterClient](./kibana-plugin-server.scopedclusterclient.md) | Serves the same purpose as "normal" <code>ClusterClient</code> but exposes additional <code>callAsCurrentUser</code> method that doesn't use credentials of the Kibana internal user (as <code>callAsInternalUser</code> does) to request Elasticsearch API, but rather passes HTTP headers extracted from the current user request to the API |
+
+## Interfaces
+
+|  Interface | Description |
+|  --- | --- |
+|  [AuthResultData](./kibana-plugin-server.authresultdata.md) | Result of an incoming request authentication. |
+|  [AuthToolkit](./kibana-plugin-server.authtoolkit.md) | A tool set defining an outcome of Auth interceptor for incoming request. |
+|  [CallAPIOptions](./kibana-plugin-server.callapioptions.md) | The set of options that defines how API call should be made and result be processed. |
+|  [CoreSetup](./kibana-plugin-server.coresetup.md) | Context passed to the plugins <code>setup</code> method. |
+|  [CoreStart](./kibana-plugin-server.corestart.md) | Context passed to the plugins <code>start</code> method. |
+|  [DiscoveredPlugin](./kibana-plugin-server.discoveredplugin.md) | Small container object used to expose information about discovered plugins that may or may not have been started. |
+|  [ElasticsearchError](./kibana-plugin-server.elasticsearcherror.md) |  |
+|  [ElasticsearchServiceSetup](./kibana-plugin-server.elasticsearchservicesetup.md) |  |
+|  [FakeRequest](./kibana-plugin-server.fakerequest.md) | Fake request object created manually by Kibana plugins. |
+|  [HttpServiceSetup](./kibana-plugin-server.httpservicesetup.md) |  |
+|  [HttpServiceStart](./kibana-plugin-server.httpservicestart.md) |  |
+|  [InternalCoreStart](./kibana-plugin-server.internalcorestart.md) |  |
+|  [KibanaRequestRoute](./kibana-plugin-server.kibanarequestroute.md) | Request specific route information exposed to a handler. |
+|  [Logger](./kibana-plugin-server.logger.md) | Logger exposes all the necessary methods to log any type of information and this is the interface used by the logging consumers including plugins. |
+|  [LoggerFactory](./kibana-plugin-server.loggerfactory.md) | The single purpose of <code>LoggerFactory</code> interface is to define a way to retrieve a context-based logger instance. |
+|  [LogMeta](./kibana-plugin-server.logmeta.md) | Contextual metadata |
+|  [OnPostAuthToolkit](./kibana-plugin-server.onpostauthtoolkit.md) | A tool set defining an outcome of OnPostAuth interceptor for incoming request. |
+|  [OnPreAuthToolkit](./kibana-plugin-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |
+|  [Plugin](./kibana-plugin-server.plugin.md) | The interface that should be returned by a <code>PluginInitializer</code>. |
+|  [PluginInitializerContext](./kibana-plugin-server.plugininitializercontext.md) | Context that's available to plugins during initialization stage. |
+|  [PluginsServiceSetup](./kibana-plugin-server.pluginsservicesetup.md) |  |
+|  [PluginsServiceStart](./kibana-plugin-server.pluginsservicestart.md) |  |
+|  [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) | A raw document as represented directly in the saved object index. |
+|  [RouteConfigOptions](./kibana-plugin-server.routeconfigoptions.md) | Route specific configuration. |
+|  [SavedObject](./kibana-plugin-server.savedobject.md) |  |
+|  [SavedObjectAttributes](./kibana-plugin-server.savedobjectattributes.md) |  |
+|  [SavedObjectReference](./kibana-plugin-server.savedobjectreference.md) | A reference to another saved object. |
+|  [SavedObjectsBaseOptions](./kibana-plugin-server.savedobjectsbaseoptions.md) |  |
+|  [SavedObjectsBulkCreateObject](./kibana-plugin-server.savedobjectsbulkcreateobject.md) |  |
+|  [SavedObjectsBulkGetObject](./kibana-plugin-server.savedobjectsbulkgetobject.md) |  |
+|  [SavedObjectsBulkResponse](./kibana-plugin-server.savedobjectsbulkresponse.md) |  |
+|  [SavedObjectsClientWrapperOptions](./kibana-plugin-server.savedobjectsclientwrapperoptions.md) |  |
+|  [SavedObjectsCreateOptions](./kibana-plugin-server.savedobjectscreateoptions.md) |  |
+|  [SavedObjectsFindOptions](./kibana-plugin-server.savedobjectsfindoptions.md) |  |
+|  [SavedObjectsFindResponse](./kibana-plugin-server.savedobjectsfindresponse.md) |  |
+|  [SavedObjectsMigrationVersion](./kibana-plugin-server.savedobjectsmigrationversion.md) | A dictionary of saved object type -<!-- -->&gt; version used to determine what migrations need to be applied to a saved object. |
+|  [SavedObjectsService](./kibana-plugin-server.savedobjectsservice.md) |  |
+|  [SavedObjectsUpdateOptions](./kibana-plugin-server.savedobjectsupdateoptions.md) |  |
+|  [SavedObjectsUpdateResponse](./kibana-plugin-server.savedobjectsupdateresponse.md) |  |
+|  [SessionStorage](./kibana-plugin-server.sessionstorage.md) | Provides an interface to store and retrieve data across requests. |
+|  [SessionStorageFactory](./kibana-plugin-server.sessionstoragefactory.md) | SessionStorage factory to bind one to an incoming request |
+
+## Type Aliases
+
+|  Type Alias | Description |
+|  --- | --- |
+|  [APICaller](./kibana-plugin-server.apicaller.md) |  |
+|  [AuthenticationHandler](./kibana-plugin-server.authenticationhandler.md) |  |
+|  [AuthHeaders](./kibana-plugin-server.authheaders.md) | Auth Headers map |
+|  [ElasticsearchClientConfig](./kibana-plugin-server.elasticsearchclientconfig.md) |  |
+|  [GetAuthHeaders](./kibana-plugin-server.getauthheaders.md) | Get headers to authenticate a user against Elasticsearch. |
+|  [Headers](./kibana-plugin-server.headers.md) |  |
+|  [LegacyRequest](./kibana-plugin-server.legacyrequest.md) | Support Legacy platform request for the period of migration. |
+|  [OnPostAuthHandler](./kibana-plugin-server.onpostauthhandler.md) |  |
+|  [OnPreAuthHandler](./kibana-plugin-server.onpreauthhandler.md) |  |
+|  [PluginInitializer](./kibana-plugin-server.plugininitializer.md) | The <code>plugin</code> export at the root of a plugin's <code>server</code> directory should conform to this interface. |
+|  [PluginName](./kibana-plugin-server.pluginname.md) | Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays that use it as a key or value more obvious. |
+|  [RecursiveReadonly](./kibana-plugin-server.recursivereadonly.md) |  |
+|  [RouteMethod](./kibana-plugin-server.routemethod.md) | The set of common HTTP methods supported by Kibana routing. |
+|  [SavedObjectsClientContract](./kibana-plugin-server.savedobjectsclientcontract.md) | \#\# SavedObjectsClient errors<!-- -->Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:<!-- -->1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md)<!-- -->Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the <code>isXYZError()</code> helpers exposed at <code>SavedObjectsErrorHelpers</code> should be used to understand and manage error responses from the <code>SavedObjectsClient</code>.<!-- -->Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for <code>error.body.error.type</code> or doing substring checks on <code>error.body.error.reason</code>, just use the helpers to understand the meaning of the error:<!-- -->\`\`\`<!-- -->js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }<!-- -->if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }<!-- -->// always rethrow the error unless you handle it throw error; \`\`\`<!-- -->\#\#\# 404s from missing index<!-- -->From the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.<!-- -->At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.<!-- -->From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.<!-- -->\#\#\# 503s from missing index<!-- -->Unlike all other methods, create requests are supposed to succeed even when the Kibana index does not exist because it will be automatically created by elasticsearch. When that is not the case it is because Elasticsearch's <code>action.auto_create_index</code> setting prevents it from being created automatically so we throw a special 503 with the intention of informing the user that their Elasticsearch settings need to be updated.<!-- -->See [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |
+|  [SavedObjectsClientWrapperFactory](./kibana-plugin-server.savedobjectsclientwrapperfactory.md) |  |
+
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._id.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._id.md
new file mode 100644
index 00000000000000..92c088060e2a1c
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._id.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_id](./kibana-plugin-server.rawsavedobjectdoc._id.md)
+
+## RawSavedObjectDoc.\_id property
+
+<b>Signature:</b>
+
+```typescript
+_id: string;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md
new file mode 100644
index 00000000000000..9892326ad58d15
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_primary\_term](./kibana-plugin-server.rawsavedobjectdoc._primary_term.md)
+
+## RawSavedObjectDoc.\_primary\_term property
+
+<b>Signature:</b>
+
+```typescript
+_primary_term?: number;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md
new file mode 100644
index 00000000000000..fdfac53065b677
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_seq\_no](./kibana-plugin-server.rawsavedobjectdoc._seq_no.md)
+
+## RawSavedObjectDoc.\_seq\_no property
+
+<b>Signature:</b>
+
+```typescript
+_seq_no?: number;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md
new file mode 100644
index 00000000000000..17e1c144b62231
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_source](./kibana-plugin-server.rawsavedobjectdoc._source.md)
+
+## RawSavedObjectDoc.\_source property
+
+<b>Signature:</b>
+
+```typescript
+_source: any;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._type.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._type.md
new file mode 100644
index 00000000000000..d399663a1fdc11
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._type.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_type](./kibana-plugin-server.rawsavedobjectdoc._type.md)
+
+## RawSavedObjectDoc.\_type property
+
+<b>Signature:</b>
+
+```typescript
+_type?: string;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md
new file mode 100644
index 00000000000000..aa2baf9d58e365
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md
@@ -0,0 +1,24 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md)
+
+## RawSavedObjectDoc interface
+
+A raw document as represented directly in the saved object index.
+
+<b>Signature:</b>
+
+```typescript
+export interface RawDoc 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [\_id](./kibana-plugin-server.rawsavedobjectdoc._id.md) | <code>string</code> |  |
+|  [\_primary\_term](./kibana-plugin-server.rawsavedobjectdoc._primary_term.md) | <code>number</code> |  |
+|  [\_seq\_no](./kibana-plugin-server.rawsavedobjectdoc._seq_no.md) | <code>number</code> |  |
+|  [\_source](./kibana-plugin-server.rawsavedobjectdoc._source.md) | <code>any</code> |  |
+|  [\_type](./kibana-plugin-server.rawsavedobjectdoc._type.md) | <code>string</code> |  |
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsschema.(constructor).md b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.(constructor).md
new file mode 100644
index 00000000000000..abac3bc88fac18
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.(constructor).md
@@ -0,0 +1,20 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md) &gt; [(constructor)](./kibana-plugin-server.savedobjectsschema.(constructor).md)
+
+## SavedObjectsSchema.(constructor)
+
+Constructs a new instance of the `SavedObjectsSchema` class
+
+<b>Signature:</b>
+
+```typescript
+constructor(schemaDefinition?: SavedObjectsSchemaDefinition);
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  schemaDefinition | <code>SavedObjectsSchemaDefinition</code> |  |
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsschema.getindexfortype.md b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.getindexfortype.md
new file mode 100644
index 00000000000000..3c9b810cfe1a69
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.getindexfortype.md
@@ -0,0 +1,22 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md) &gt; [getIndexForType](./kibana-plugin-server.savedobjectsschema.getindexfortype.md)
+
+## SavedObjectsSchema.getIndexForType() method
+
+<b>Signature:</b>
+
+```typescript
+getIndexForType(type: string): string | undefined;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  type | <code>string</code> |  |
+
+<b>Returns:</b>
+
+`string | undefined`
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsschema.ishiddentype.md b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.ishiddentype.md
new file mode 100644
index 00000000000000..f67b12a4d14c3d
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.ishiddentype.md
@@ -0,0 +1,22 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md) &gt; [isHiddenType](./kibana-plugin-server.savedobjectsschema.ishiddentype.md)
+
+## SavedObjectsSchema.isHiddenType() method
+
+<b>Signature:</b>
+
+```typescript
+isHiddenType(type: string): boolean;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  type | <code>string</code> |  |
+
+<b>Returns:</b>
+
+`boolean`
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsschema.isnamespaceagnostic.md b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.isnamespaceagnostic.md
new file mode 100644
index 00000000000000..2ca0abd7e4aa7e
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.isnamespaceagnostic.md
@@ -0,0 +1,22 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md) &gt; [isNamespaceAgnostic](./kibana-plugin-server.savedobjectsschema.isnamespaceagnostic.md)
+
+## SavedObjectsSchema.isNamespaceAgnostic() method
+
+<b>Signature:</b>
+
+```typescript
+isNamespaceAgnostic(type: string): boolean;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  type | <code>string</code> |  |
+
+<b>Returns:</b>
+
+`boolean`
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsschema.md b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.md
new file mode 100644
index 00000000000000..1b9cb2ad94c22b
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsschema.md
@@ -0,0 +1,26 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md)
+
+## SavedObjectsSchema class
+
+<b>Signature:</b>
+
+```typescript
+export declare class SavedObjectsSchema 
+```
+
+## Constructors
+
+|  Constructor | Modifiers | Description |
+|  --- | --- | --- |
+|  [(constructor)(schemaDefinition)](./kibana-plugin-server.savedobjectsschema.(constructor).md) |  | Constructs a new instance of the <code>SavedObjectsSchema</code> class |
+
+## Methods
+
+|  Method | Modifiers | Description |
+|  --- | --- | --- |
+|  [getIndexForType(type)](./kibana-plugin-server.savedobjectsschema.getindexfortype.md) |  |  |
+|  [isHiddenType(type)](./kibana-plugin-server.savedobjectsschema.ishiddentype.md) |  |  |
+|  [isNamespaceAgnostic(type)](./kibana-plugin-server.savedobjectsschema.isnamespaceagnostic.md) |  |  |
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.(constructor).md b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.(constructor).md
new file mode 100644
index 00000000000000..6524ff3e17caf8
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.(constructor).md
@@ -0,0 +1,20 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) &gt; [(constructor)](./kibana-plugin-server.savedobjectsserializer.(constructor).md)
+
+## SavedObjectsSerializer.(constructor)
+
+Constructs a new instance of the `SavedObjectsSerializer` class
+
+<b>Signature:</b>
+
+```typescript
+constructor(schema: SavedObjectsSchema);
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  schema | <code>SavedObjectsSchema</code> |  |
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.generaterawid.md b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.generaterawid.md
new file mode 100644
index 00000000000000..4705f48a201aee
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.generaterawid.md
@@ -0,0 +1,26 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) &gt; [generateRawId](./kibana-plugin-server.savedobjectsserializer.generaterawid.md)
+
+## SavedObjectsSerializer.generateRawId() method
+
+Given a saved object type and id, generates the compound id that is stored in the raw document.
+
+<b>Signature:</b>
+
+```typescript
+generateRawId(namespace: string | undefined, type: string, id?: string): string;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  namespace | <code>string &#124; undefined</code> |  |
+|  type | <code>string</code> |  |
+|  id | <code>string</code> |  |
+
+<b>Returns:</b>
+
+`string`
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.israwsavedobject.md b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.israwsavedobject.md
new file mode 100644
index 00000000000000..e190e7bce8c011
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.israwsavedobject.md
@@ -0,0 +1,24 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) &gt; [isRawSavedObject](./kibana-plugin-server.savedobjectsserializer.israwsavedobject.md)
+
+## SavedObjectsSerializer.isRawSavedObject() method
+
+Determines whether or not the raw document can be converted to a saved object.
+
+<b>Signature:</b>
+
+```typescript
+isRawSavedObject(rawDoc: RawDoc): any;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  rawDoc | <code>RawDoc</code> |  |
+
+<b>Returns:</b>
+
+`any`
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.md b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.md
new file mode 100644
index 00000000000000..205e29cb0727da
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.md
@@ -0,0 +1,27 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md)
+
+## SavedObjectsSerializer class
+
+<b>Signature:</b>
+
+```typescript
+export declare class SavedObjectsSerializer 
+```
+
+## Constructors
+
+|  Constructor | Modifiers | Description |
+|  --- | --- | --- |
+|  [(constructor)(schema)](./kibana-plugin-server.savedobjectsserializer.(constructor).md) |  | Constructs a new instance of the <code>SavedObjectsSerializer</code> class |
+
+## Methods
+
+|  Method | Modifiers | Description |
+|  --- | --- | --- |
+|  [generateRawId(namespace, type, id)](./kibana-plugin-server.savedobjectsserializer.generaterawid.md) |  | Given a saved object type and id, generates the compound id that is stored in the raw document. |
+|  [isRawSavedObject(rawDoc)](./kibana-plugin-server.savedobjectsserializer.israwsavedobject.md) |  | Determines whether or not the raw document can be converted to a saved object. |
+|  [rawToSavedObject(doc)](./kibana-plugin-server.savedobjectsserializer.rawtosavedobject.md) |  | Converts a document from the format that is stored in elasticsearch to the saved object client format. |
+|  [savedObjectToRaw(savedObj)](./kibana-plugin-server.savedobjectsserializer.savedobjecttoraw.md) |  | Converts a document from the saved object client format to the format that is stored in elasticsearch. |
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.rawtosavedobject.md b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.rawtosavedobject.md
new file mode 100644
index 00000000000000..b36cdb3be64da9
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.rawtosavedobject.md
@@ -0,0 +1,24 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) &gt; [rawToSavedObject](./kibana-plugin-server.savedobjectsserializer.rawtosavedobject.md)
+
+## SavedObjectsSerializer.rawToSavedObject() method
+
+Converts a document from the format that is stored in elasticsearch to the saved object client format.
+
+<b>Signature:</b>
+
+```typescript
+rawToSavedObject(doc: RawDoc): SanitizedSavedObjectDoc;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  doc | <code>RawDoc</code> |  |
+
+<b>Returns:</b>
+
+`SanitizedSavedObjectDoc`
+
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.savedobjecttoraw.md b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.savedobjecttoraw.md
new file mode 100644
index 00000000000000..4854a97a845b89
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsserializer.savedobjecttoraw.md
@@ -0,0 +1,24 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) &gt; [savedObjectToRaw](./kibana-plugin-server.savedobjectsserializer.savedobjecttoraw.md)
+
+## SavedObjectsSerializer.savedObjectToRaw() method
+
+Converts a document from the saved object client format to the format that is stored in elasticsearch.
+
+<b>Signature:</b>
+
+```typescript
+savedObjectToRaw(savedObj: SanitizedSavedObjectDoc): RawDoc;
+```
+
+## Parameters
+
+|  Parameter | Type | Description |
+|  --- | --- | --- |
+|  savedObj | <code>SanitizedSavedObjectDoc</code> |  |
+
+<b>Returns:</b>
+
+`RawDoc`
+
diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md
index a6fbfebf9d9470..a99161b89664e1 100644
--- a/src/core/server/server.api.md
+++ b/src/core/server/server.api.md
@@ -378,6 +378,22 @@ export interface PluginsServiceStart {
     contracts: Map<PluginName, unknown>;
 }
 
+// Warning: (ae-missing-release-tag) "RawDoc" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// 
+// @public
+export interface RawSavedObjectDoc {
+    // (undocumented)
+    _id: string;
+    // (undocumented)
+    _primary_term?: number;
+    // (undocumented)
+    _seq_no?: number;
+    // (undocumented)
+    _source: any;
+    // (undocumented)
+    _type?: string;
+}
+
 // Warning: (ae-forgotten-export) The symbol "RecursiveReadonlyArray" needs to be exported by the entry point index.d.ts
 // 
 // @public (undocumented)
@@ -634,6 +650,32 @@ export interface SavedObjectsMigrationVersion {
     [pluginName: string]: string;
 }
 
+// Warning: (ae-missing-release-tag) "SavedObjectsSchema" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// 
+// @public (undocumented)
+export class SavedObjectsSchema {
+    // Warning: (ae-forgotten-export) The symbol "SavedObjectsSchemaDefinition" needs to be exported by the entry point index.d.ts
+    constructor(schemaDefinition?: SavedObjectsSchemaDefinition);
+    // (undocumented)
+    getIndexForType(type: string): string | undefined;
+    // (undocumented)
+    isHiddenType(type: string): boolean;
+    // (undocumented)
+    isNamespaceAgnostic(type: string): boolean;
+}
+
+// Warning: (ae-missing-release-tag) "SavedObjectsSerializer" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// 
+// @public (undocumented)
+export class SavedObjectsSerializer {
+    constructor(schema: SavedObjectsSchema);
+    generateRawId(namespace: string | undefined, type: string, id?: string): string;
+    isRawSavedObject(rawDoc: RawSavedObjectDoc): any;
+    // Warning: (ae-forgotten-export) The symbol "SanitizedSavedObjectDoc" needs to be exported by the entry point index.d.ts
+    rawToSavedObject(doc: RawSavedObjectDoc): SanitizedSavedObjectDoc;
+    savedObjectToRaw(savedObj: SanitizedSavedObjectDoc): RawSavedObjectDoc;
+    }
+
 // @public (undocumented)
 export interface SavedObjectsService<Request = any> {
     // Warning: (ae-forgotten-export) The symbol "ScopedSavedObjectsClientProvider" needs to be exported by the entry point index.d.ts

From 732a2b044d7ba822a4d1b77b49dce0c4c5504717 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Wed, 17 Jul 2019 11:45:53 -0400
Subject: [PATCH 42/48] PR feedback

---
 x-pack/legacy/plugins/task_manager/index.js        |  2 +-
 .../task_manager/{migrations.js => migrations.ts}  | 14 ++++++++------
 2 files changed, 9 insertions(+), 7 deletions(-)
 rename x-pack/legacy/plugins/task_manager/{migrations.js => migrations.ts} (58%)

diff --git a/x-pack/legacy/plugins/task_manager/index.js b/x-pack/legacy/plugins/task_manager/index.js
index 8365e03ae594cb..ba92e9db50601a 100644
--- a/x-pack/legacy/plugins/task_manager/index.js
+++ b/x-pack/legacy/plugins/task_manager/index.js
@@ -7,7 +7,7 @@
 import { SavedObjectsSerializer, SavedObjectsSchema } from '../../../../src/core/server';
 import { TaskManager } from './task_manager';
 import mappings from './mappings.json';
-import migrations from './migrations';
+import { migrations } from './migrations';
 import { TASK_MANAGER_INDEX } from './constants';
 
 export function taskManager(kibana) {
diff --git a/x-pack/legacy/plugins/task_manager/migrations.js b/x-pack/legacy/plugins/task_manager/migrations.ts
similarity index 58%
rename from x-pack/legacy/plugins/task_manager/migrations.js
rename to x-pack/legacy/plugins/task_manager/migrations.ts
index a9027a4dbd54d5..dd6651fddb90a6 100644
--- a/x-pack/legacy/plugins/task_manager/migrations.js
+++ b/x-pack/legacy/plugins/task_manager/migrations.ts
@@ -4,11 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export default {
+import { SavedObject } from 'src/core/server';
+
+export const migrations = {
   task: {
-    '7.4.0': (doc) => {
-      doc.updated_at = new Date().toISOString();
-      return doc;
-    }
-  }
+    '7.4.0': (doc: SavedObject) => ({
+      ...doc,
+      updated_at: new Date().toISOString(),
+    }),
+  },
 };

From 53c57a988c936f58ef8bf279a0a402512cf4ae24 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 18 Jul 2019 08:24:25 -0400
Subject: [PATCH 43/48] Apply PR feedback

---
 .../plugins/task_manager/task_runner.ts       | 22 ++++++++++---------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 50286d531e1c11..882e145518f9ab 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -12,7 +12,7 @@
 
 import Joi from 'joi';
 import Boom from 'boom';
-import { minutesFromNow, intervalFromDate, intervalFromNow } from './lib/intervals';
+import { intervalFromDate, intervalFromNow } from './lib/intervals';
 import { Logger } from './lib/logger';
 import { BeforeRunFunction } from './lib/middleware';
 import {
@@ -25,6 +25,8 @@ import {
   validateRunResult,
 } from './task';
 
+const defaultBackoffPerFailure = 5 * 60 * 1000;
+
 export interface TaskRunner {
   numWorkers: number;
   isExpired: boolean;
@@ -178,10 +180,7 @@ export class TaskManagerRunner implements TaskRunner {
         startedAt: now,
         attempts,
         retryAt: new Date(
-          timeoutDate.getTime() +
-            (this.definition.getRetryDelay
-              ? this.definition.getRetryDelay(attempts, Boom.clientTimeout()) * 1000
-              : attempts * 5 * 60 * 1000) // incrementally backs off an extra 5m per failure
+          timeoutDate.getTime() + this.getRetryDelay(attempts, Boom.clientTimeout())
         ),
       });
 
@@ -234,11 +233,7 @@ export class TaskManagerRunner implements TaskRunner {
       runAt = result.runAt;
     } else if (result.error) {
       // when result.error is truthy, then we're retrying because it failed
-      runAt = this.definition.getRetryDelay
-        ? new Date(
-            Date.now() + this.definition.getRetryDelay(this.instance.attempts, result.error) * 1000
-          )
-        : minutesFromNow(this.instance.attempts * 5); // incrementally backs off an extra 5m per failure
+      runAt = new Date(Date.now() + this.getRetryDelay(this.instance.attempts, result.error));
     } else {
       runAt = intervalFromDate(startedAt, this.instance.interval)!;
     }
@@ -290,6 +285,13 @@ export class TaskManagerRunner implements TaskRunner {
     const maxAttempts = this.definition.maxAttempts || this.store.maxAttempts;
     return this.instance.attempts < maxAttempts ? 'idle' : 'failed';
   }
+
+  private getRetryDelay(attempts: number, error: any) {
+    if (this.definition.getRetryDelay) {
+      return this.definition.getRetryDelay(attempts, error) * 1000;
+    }
+    return attempts * defaultBackoffPerFailure;
+  }
 }
 
 function sanitizeInstance(instance: ConcreteTaskInstance): ConcreteTaskInstance {

From bbdc7e2df0c9f78e660fbb0cafad83b928805617 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 18 Jul 2019 09:08:37 -0400
Subject: [PATCH 44/48] Apply PR feedback pt2

---
 .../plugins/task_manager/lib/intervals.test.ts       | 10 ++--------
 .../legacy/plugins/task_manager/task_runner.test.ts  | 12 +++++-------
 2 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
index 7e8b2dbac69bf4..382691471943fd 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
@@ -5,6 +5,7 @@
  */
 
 import _ from 'lodash';
+import sinon from 'sinon';
 import {
   assertValidInterval,
   intervalFromNow,
@@ -16,14 +17,7 @@ import {
 } from './intervals';
 
 const mockedNow = new Date('2019-06-03T18:55:25.982Z');
-(global as any).Date = class Date extends global.Date {
-  static now() {
-    return mockedNow.getTime();
-  }
-  valueOf() {
-    return mockedNow.valueOf();
-  }
-};
+sinon.useFakeTimers(mockedNow.valueOf());
 
 describe('taskIntervals', () => {
   describe('assertValidInterval', () => {
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index af5e864edf6b42..fce78a5adf3fc4 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -11,11 +11,7 @@ import { ConcreteTaskInstance } from './task';
 import { TaskManagerRunner } from './task_runner';
 
 const mockedNow = new Date('2019-06-03T18:55:25.982Z');
-(global as any).Date = class Date extends global.Date {
-  static now() {
-    return mockedNow.getTime();
-  }
-};
+const clock = sinon.useFakeTimers(mockedNow.valueOf());
 
 describe('TaskManagerRunner', () => {
   test('provides details about the task that is running', () => {
@@ -182,7 +178,9 @@ describe('TaskManagerRunner', () => {
         bar: {
           createTaskRunner: () => ({
             async run() {
-              await new Promise(r => setTimeout(r, 1000));
+              const promise = new Promise(r => setTimeout(r, 1000));
+              clock.tick(1000);
+              await promise;
             },
             async cancel() {
               wasCancelled = true;
@@ -193,7 +191,7 @@ describe('TaskManagerRunner', () => {
     });
 
     const promise = runner.run();
-    await new Promise(r => setInterval(r, 1));
+    await Promise.resolve();
     await runner.cancel();
     await promise;
 

From 61444ca9f847557e54bcebac47cb7359a4659848 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 18 Jul 2019 09:10:01 -0400
Subject: [PATCH 45/48] Apply PR feedback pt3

---
 x-pack/legacy/plugins/task_manager/lib/intervals.ts         | 4 ++--
 x-pack/legacy/plugins/task_manager/task_runner.test.ts      | 6 +++---
 x-pack/legacy/plugins/task_manager/task_runner.ts           | 2 +-
 .../public/lib/helper/charts/is_within_current_date.ts      | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
index 92b68a7e6a7596..9009be5f78220b 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.ts
@@ -51,7 +51,7 @@ export function intervalFromDate(date: Date, interval?: string): Date | undefine
  * @param mins The number of mintues from now
  */
 export function minutesFromNow(mins: number): Date {
-  return minutesFromDate(new Date(Date.now()), mins);
+  return minutesFromDate(new Date(), mins);
 }
 
 /**
@@ -74,7 +74,7 @@ export function minutesFromDate(date: Date, mins: number): Date {
  * @param secs The number of seconds from now
  */
 export function secondsFromNow(secs: number): Date {
-  return secondsFromDate(new Date(Date.now()), secs);
+  return secondsFromDate(new Date(), secs);
 }
 
 /**
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index fce78a5adf3fc4..64b49fe434fc66 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -82,7 +82,7 @@ describe('TaskManagerRunner', () => {
       instance: {
         interval: '10m',
         status: 'running',
-        startedAt: new Date(Date.now()),
+        startedAt: new Date(),
       },
       definitions: {
         bar: {
@@ -407,8 +407,8 @@ describe('TaskManagerRunner', () => {
           taskType: 'bar',
           sequenceNumber: 32,
           primaryTerm: 32,
-          runAt: new Date(Date.now()),
-          scheduledAt: new Date(Date.now()),
+          runAt: new Date(),
+          scheduledAt: new Date(),
           startedAt: null,
           retryAt: null,
           attempts: 0,
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.ts b/x-pack/legacy/plugins/task_manager/task_runner.ts
index 882e145518f9ab..49936488739031 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.ts
@@ -169,7 +169,7 @@ export class TaskManagerRunner implements TaskRunner {
   public async claimOwnership(): Promise<boolean> {
     const VERSION_CONFLICT_STATUS = 409;
     const attempts = this.instance.attempts + 1;
-    const now = new Date(Date.now());
+    const now = new Date();
 
     const timeoutDate = intervalFromNow(this.definition.timeout!)!;
 
diff --git a/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts b/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts
index 8ef33ed4161fdb..830b5db9068ec0 100644
--- a/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts
+++ b/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts
@@ -23,7 +23,7 @@
  * isWithinCurrentDate(from, to); // returns true
  */
 export const isWithinCurrentDate = (dateRangeStart: number, dateRangeEnd: number) => {
-  const today = new Date(Date.now());
+  const today = new Date();
   const min = today.setHours(0, 0, 0, 0).valueOf();
   const max = today.setHours(23, 59, 59, 999).valueOf();
   return dateRangeStart > min && dateRangeStart < max && dateRangeEnd < max;

From d07fee2129903d946c733d69342b95a946611db9 Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 18 Jul 2019 09:37:58 -0400
Subject: [PATCH 46/48] Apply PR feedback pt4

---
 .../legacy/plugins/task_manager/lib/intervals.test.ts |  9 +++++++--
 .../legacy/plugins/task_manager/task_runner.test.ts   | 11 ++++++++---
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
index 382691471943fd..a8e186c397d76d 100644
--- a/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
+++ b/x-pack/legacy/plugins/task_manager/lib/intervals.test.ts
@@ -16,8 +16,13 @@ import {
   secondsFromDate,
 } from './intervals';
 
-const mockedNow = new Date('2019-06-03T18:55:25.982Z');
-sinon.useFakeTimers(mockedNow.valueOf());
+let fakeTimer: sinon.SinonFakeTimers;
+
+beforeAll(() => {
+  fakeTimer = sinon.useFakeTimers();
+});
+
+afterAll(() => fakeTimer.restore());
 
 describe('taskIntervals', () => {
   describe('assertValidInterval', () => {
diff --git a/x-pack/legacy/plugins/task_manager/task_runner.test.ts b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
index 64b49fe434fc66..95dead2013a269 100644
--- a/x-pack/legacy/plugins/task_manager/task_runner.test.ts
+++ b/x-pack/legacy/plugins/task_manager/task_runner.test.ts
@@ -10,8 +10,13 @@ import { minutesFromNow, secondsFromNow } from './lib/intervals';
 import { ConcreteTaskInstance } from './task';
 import { TaskManagerRunner } from './task_runner';
 
-const mockedNow = new Date('2019-06-03T18:55:25.982Z');
-const clock = sinon.useFakeTimers(mockedNow.valueOf());
+let fakeTimer: sinon.SinonFakeTimers;
+
+beforeAll(() => {
+  fakeTimer = sinon.useFakeTimers();
+});
+
+afterAll(() => fakeTimer.restore());
 
 describe('TaskManagerRunner', () => {
   test('provides details about the task that is running', () => {
@@ -179,7 +184,7 @@ describe('TaskManagerRunner', () => {
           createTaskRunner: () => ({
             async run() {
               const promise = new Promise(r => setTimeout(r, 1000));
-              clock.tick(1000);
+              fakeTimer.tick(1000);
               await promise;
             },
             async cancel() {

From ff78cf6a10c7560d963ca55ca55bbfe7b4308a5b Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Thu, 18 Jul 2019 10:16:21 -0400
Subject: [PATCH 47/48] Fix feedback pt3

---
 .../uptime/public/lib/helper/charts/is_within_current_date.ts   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts b/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts
index 830b5db9068ec0..8ef33ed4161fdb 100644
--- a/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts
+++ b/x-pack/legacy/plugins/uptime/public/lib/helper/charts/is_within_current_date.ts
@@ -23,7 +23,7 @@
  * isWithinCurrentDate(from, to); // returns true
  */
 export const isWithinCurrentDate = (dateRangeStart: number, dateRangeEnd: number) => {
-  const today = new Date();
+  const today = new Date(Date.now());
   const min = today.setHours(0, 0, 0, 0).valueOf();
   const max = today.setHours(23, 59, 59, 999).valueOf();
   return dateRangeStart > min && dateRangeStart < max && dateRangeEnd < max;

From ac490fd4e9468213138566bfedbb845c81439dac Mon Sep 17 00:00:00 2001
From: Mike Cote <mikecote@users.noreply.github.com>
Date: Tue, 23 Jul 2019 14:18:40 -0400
Subject: [PATCH 48/48] Rename RawSavedObjectDoc to SavedObjectsRawDoc

---
 .../core/server/kibana-plugin-server.md       | 182 +++++++++---------
 ...-server.rawsavedobjectdoc._primary_term.md |  11 --
 ...plugin-server.rawsavedobjectdoc._seq_no.md |  11 --
 ...plugin-server.rawsavedobjectdoc._source.md |  11 --
 .../kibana-plugin-server.rawsavedobjectdoc.md |  24 ---
 ...a-plugin-server.savedobjectsrawdoc._id.md} |   4 +-
 ...server.savedobjectsrawdoc._primary_term.md |  11 ++
 ...lugin-server.savedobjectsrawdoc._seq_no.md |  11 ++
 ...lugin-server.savedobjectsrawdoc._source.md |  11 ++
 ...plugin-server.savedobjectsrawdoc._type.md} |   4 +-
 ...kibana-plugin-server.savedobjectsrawdoc.md |  24 +++
 src/core/server/index.ts                      |   2 +-
 src/core/server/saved_objects/index.ts        |   2 +-
 src/core/server/server.api.md                 |  38 ++--
 .../legacy/plugins/task_manager/task_store.ts |   4 +-
 15 files changed, 175 insertions(+), 175 deletions(-)
 delete mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md
 delete mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md
 delete mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md
 delete mode 100644 docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md
 rename docs/development/core/server/{kibana-plugin-server.rawsavedobjectdoc._id.md => kibana-plugin-server.savedobjectsrawdoc._id.md} (50%)
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._primary_term.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._seq_no.md
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._source.md
 rename docs/development/core/server/{kibana-plugin-server.rawsavedobjectdoc._type.md => kibana-plugin-server.savedobjectsrawdoc._type.md} (50%)
 create mode 100644 docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc.md

diff --git a/docs/development/core/server/kibana-plugin-server.md b/docs/development/core/server/kibana-plugin-server.md
index d8d53421d68748..0c106ecfb5e70f 100644
--- a/docs/development/core/server/kibana-plugin-server.md
+++ b/docs/development/core/server/kibana-plugin-server.md
@@ -1,91 +1,91 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md)
-
-## kibana-plugin-server package
-
-The Kibana Core APIs for server-side plugins.
-
-A plugin's `server/index` file must contain a named import, `plugin`<!-- -->, that implements [PluginInitializer](./kibana-plugin-server.plugininitializer.md) which returns an object that implements [Plugin](./kibana-plugin-server.plugin.md)<!-- -->.
-
-The plugin integrates with the core system via lifecycle events: `setup`<!-- -->, `start`<!-- -->, and `stop`<!-- -->. In each lifecycle method, the plugin will receive the corresponding core services available (either [CoreSetup](./kibana-plugin-server.coresetup.md) or [CoreStart](./kibana-plugin-server.corestart.md)<!-- -->) and any interfaces returned by dependency plugins' lifecycle method. Anything returned by the plugin's lifecycle method will be exposed to downstream dependencies when their corresponding lifecycle methods are invoked.
-
-## Classes
-
-|  Class | Description |
-|  --- | --- |
-|  [ClusterClient](./kibana-plugin-server.clusterclient.md) | Represents an Elasticsearch cluster API client and allows to call API on behalf of the internal Kibana user and the actual user that is derived from the request headers (via <code>asScoped(...)</code>). |
-|  [ElasticsearchErrorHelpers](./kibana-plugin-server.elasticsearcherrorhelpers.md) | Helpers for working with errors returned from the Elasticsearch service.Since the internal data of errors are subject to change, consumers of the Elasticsearch service should always use these helpers to classify errors instead of checking error internals such as <code>body.error.header[WWW-Authenticate]</code> |
-|  [KibanaRequest](./kibana-plugin-server.kibanarequest.md) | Kibana specific abstraction for an incoming request. |
-|  [Router](./kibana-plugin-server.router.md) |  |
-|  [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |  |
-|  [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md) |  |
-|  [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) |  |
-|  [ScopedClusterClient](./kibana-plugin-server.scopedclusterclient.md) | Serves the same purpose as "normal" <code>ClusterClient</code> but exposes additional <code>callAsCurrentUser</code> method that doesn't use credentials of the Kibana internal user (as <code>callAsInternalUser</code> does) to request Elasticsearch API, but rather passes HTTP headers extracted from the current user request to the API |
-
-## Interfaces
-
-|  Interface | Description |
-|  --- | --- |
-|  [AuthResultData](./kibana-plugin-server.authresultdata.md) | Result of an incoming request authentication. |
-|  [AuthToolkit](./kibana-plugin-server.authtoolkit.md) | A tool set defining an outcome of Auth interceptor for incoming request. |
-|  [CallAPIOptions](./kibana-plugin-server.callapioptions.md) | The set of options that defines how API call should be made and result be processed. |
-|  [CoreSetup](./kibana-plugin-server.coresetup.md) | Context passed to the plugins <code>setup</code> method. |
-|  [CoreStart](./kibana-plugin-server.corestart.md) | Context passed to the plugins <code>start</code> method. |
-|  [DiscoveredPlugin](./kibana-plugin-server.discoveredplugin.md) | Small container object used to expose information about discovered plugins that may or may not have been started. |
-|  [ElasticsearchError](./kibana-plugin-server.elasticsearcherror.md) |  |
-|  [ElasticsearchServiceSetup](./kibana-plugin-server.elasticsearchservicesetup.md) |  |
-|  [FakeRequest](./kibana-plugin-server.fakerequest.md) | Fake request object created manually by Kibana plugins. |
-|  [HttpServiceSetup](./kibana-plugin-server.httpservicesetup.md) |  |
-|  [HttpServiceStart](./kibana-plugin-server.httpservicestart.md) |  |
-|  [InternalCoreStart](./kibana-plugin-server.internalcorestart.md) |  |
-|  [KibanaRequestRoute](./kibana-plugin-server.kibanarequestroute.md) | Request specific route information exposed to a handler. |
-|  [Logger](./kibana-plugin-server.logger.md) | Logger exposes all the necessary methods to log any type of information and this is the interface used by the logging consumers including plugins. |
-|  [LoggerFactory](./kibana-plugin-server.loggerfactory.md) | The single purpose of <code>LoggerFactory</code> interface is to define a way to retrieve a context-based logger instance. |
-|  [LogMeta](./kibana-plugin-server.logmeta.md) | Contextual metadata |
-|  [OnPostAuthToolkit](./kibana-plugin-server.onpostauthtoolkit.md) | A tool set defining an outcome of OnPostAuth interceptor for incoming request. |
-|  [OnPreAuthToolkit](./kibana-plugin-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |
-|  [Plugin](./kibana-plugin-server.plugin.md) | The interface that should be returned by a <code>PluginInitializer</code>. |
-|  [PluginInitializerContext](./kibana-plugin-server.plugininitializercontext.md) | Context that's available to plugins during initialization stage. |
-|  [PluginsServiceSetup](./kibana-plugin-server.pluginsservicesetup.md) |  |
-|  [PluginsServiceStart](./kibana-plugin-server.pluginsservicestart.md) |  |
-|  [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) | A raw document as represented directly in the saved object index. |
-|  [RouteConfigOptions](./kibana-plugin-server.routeconfigoptions.md) | Route specific configuration. |
-|  [SavedObject](./kibana-plugin-server.savedobject.md) |  |
-|  [SavedObjectAttributes](./kibana-plugin-server.savedobjectattributes.md) |  |
-|  [SavedObjectReference](./kibana-plugin-server.savedobjectreference.md) | A reference to another saved object. |
-|  [SavedObjectsBaseOptions](./kibana-plugin-server.savedobjectsbaseoptions.md) |  |
-|  [SavedObjectsBulkCreateObject](./kibana-plugin-server.savedobjectsbulkcreateobject.md) |  |
-|  [SavedObjectsBulkGetObject](./kibana-plugin-server.savedobjectsbulkgetobject.md) |  |
-|  [SavedObjectsBulkResponse](./kibana-plugin-server.savedobjectsbulkresponse.md) |  |
-|  [SavedObjectsClientWrapperOptions](./kibana-plugin-server.savedobjectsclientwrapperoptions.md) |  |
-|  [SavedObjectsCreateOptions](./kibana-plugin-server.savedobjectscreateoptions.md) |  |
-|  [SavedObjectsFindOptions](./kibana-plugin-server.savedobjectsfindoptions.md) |  |
-|  [SavedObjectsFindResponse](./kibana-plugin-server.savedobjectsfindresponse.md) |  |
-|  [SavedObjectsMigrationVersion](./kibana-plugin-server.savedobjectsmigrationversion.md) | A dictionary of saved object type -<!-- -->&gt; version used to determine what migrations need to be applied to a saved object. |
-|  [SavedObjectsService](./kibana-plugin-server.savedobjectsservice.md) |  |
-|  [SavedObjectsUpdateOptions](./kibana-plugin-server.savedobjectsupdateoptions.md) |  |
-|  [SavedObjectsUpdateResponse](./kibana-plugin-server.savedobjectsupdateresponse.md) |  |
-|  [SessionStorage](./kibana-plugin-server.sessionstorage.md) | Provides an interface to store and retrieve data across requests. |
-|  [SessionStorageFactory](./kibana-plugin-server.sessionstoragefactory.md) | SessionStorage factory to bind one to an incoming request |
-
-## Type Aliases
-
-|  Type Alias | Description |
-|  --- | --- |
-|  [APICaller](./kibana-plugin-server.apicaller.md) |  |
-|  [AuthenticationHandler](./kibana-plugin-server.authenticationhandler.md) |  |
-|  [AuthHeaders](./kibana-plugin-server.authheaders.md) | Auth Headers map |
-|  [ElasticsearchClientConfig](./kibana-plugin-server.elasticsearchclientconfig.md) |  |
-|  [GetAuthHeaders](./kibana-plugin-server.getauthheaders.md) | Get headers to authenticate a user against Elasticsearch. |
-|  [Headers](./kibana-plugin-server.headers.md) |  |
-|  [LegacyRequest](./kibana-plugin-server.legacyrequest.md) | Support Legacy platform request for the period of migration. |
-|  [OnPostAuthHandler](./kibana-plugin-server.onpostauthhandler.md) |  |
-|  [OnPreAuthHandler](./kibana-plugin-server.onpreauthhandler.md) |  |
-|  [PluginInitializer](./kibana-plugin-server.plugininitializer.md) | The <code>plugin</code> export at the root of a plugin's <code>server</code> directory should conform to this interface. |
-|  [PluginName](./kibana-plugin-server.pluginname.md) | Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays that use it as a key or value more obvious. |
-|  [RecursiveReadonly](./kibana-plugin-server.recursivereadonly.md) |  |
-|  [RouteMethod](./kibana-plugin-server.routemethod.md) | The set of common HTTP methods supported by Kibana routing. |
-|  [SavedObjectsClientContract](./kibana-plugin-server.savedobjectsclientcontract.md) | \#\# SavedObjectsClient errors<!-- -->Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:<!-- -->1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md)<!-- -->Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the <code>isXYZError()</code> helpers exposed at <code>SavedObjectsErrorHelpers</code> should be used to understand and manage error responses from the <code>SavedObjectsClient</code>.<!-- -->Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for <code>error.body.error.type</code> or doing substring checks on <code>error.body.error.reason</code>, just use the helpers to understand the meaning of the error:<!-- -->\`\`\`<!-- -->js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }<!-- -->if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }<!-- -->// always rethrow the error unless you handle it throw error; \`\`\`<!-- -->\#\#\# 404s from missing index<!-- -->From the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.<!-- -->At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.<!-- -->From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.<!-- -->\#\#\# 503s from missing index<!-- -->Unlike all other methods, create requests are supposed to succeed even when the Kibana index does not exist because it will be automatically created by elasticsearch. When that is not the case it is because Elasticsearch's <code>action.auto_create_index</code> setting prevents it from being created automatically so we throw a special 503 with the intention of informing the user that their Elasticsearch settings need to be updated.<!-- -->See [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |
-|  [SavedObjectsClientWrapperFactory](./kibana-plugin-server.savedobjectsclientwrapperfactory.md) |  |
-
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md)
+
+## kibana-plugin-server package
+
+The Kibana Core APIs for server-side plugins.
+
+A plugin's `server/index` file must contain a named import, `plugin`<!-- -->, that implements [PluginInitializer](./kibana-plugin-server.plugininitializer.md) which returns an object that implements [Plugin](./kibana-plugin-server.plugin.md)<!-- -->.
+
+The plugin integrates with the core system via lifecycle events: `setup`<!-- -->, `start`<!-- -->, and `stop`<!-- -->. In each lifecycle method, the plugin will receive the corresponding core services available (either [CoreSetup](./kibana-plugin-server.coresetup.md) or [CoreStart](./kibana-plugin-server.corestart.md)<!-- -->) and any interfaces returned by dependency plugins' lifecycle method. Anything returned by the plugin's lifecycle method will be exposed to downstream dependencies when their corresponding lifecycle methods are invoked.
+
+## Classes
+
+|  Class | Description |
+|  --- | --- |
+|  [ClusterClient](./kibana-plugin-server.clusterclient.md) | Represents an Elasticsearch cluster API client and allows to call API on behalf of the internal Kibana user and the actual user that is derived from the request headers (via <code>asScoped(...)</code>). |
+|  [ElasticsearchErrorHelpers](./kibana-plugin-server.elasticsearcherrorhelpers.md) | Helpers for working with errors returned from the Elasticsearch service.Since the internal data of errors are subject to change, consumers of the Elasticsearch service should always use these helpers to classify errors instead of checking error internals such as <code>body.error.header[WWW-Authenticate]</code> |
+|  [KibanaRequest](./kibana-plugin-server.kibanarequest.md) | Kibana specific abstraction for an incoming request. |
+|  [Router](./kibana-plugin-server.router.md) |  |
+|  [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |  |
+|  [SavedObjectsSchema](./kibana-plugin-server.savedobjectsschema.md) |  |
+|  [SavedObjectsSerializer](./kibana-plugin-server.savedobjectsserializer.md) |  |
+|  [ScopedClusterClient](./kibana-plugin-server.scopedclusterclient.md) | Serves the same purpose as "normal" <code>ClusterClient</code> but exposes additional <code>callAsCurrentUser</code> method that doesn't use credentials of the Kibana internal user (as <code>callAsInternalUser</code> does) to request Elasticsearch API, but rather passes HTTP headers extracted from the current user request to the API |
+
+## Interfaces
+
+|  Interface | Description |
+|  --- | --- |
+|  [AuthResultData](./kibana-plugin-server.authresultdata.md) | Result of an incoming request authentication. |
+|  [AuthToolkit](./kibana-plugin-server.authtoolkit.md) | A tool set defining an outcome of Auth interceptor for incoming request. |
+|  [CallAPIOptions](./kibana-plugin-server.callapioptions.md) | The set of options that defines how API call should be made and result be processed. |
+|  [CoreSetup](./kibana-plugin-server.coresetup.md) | Context passed to the plugins <code>setup</code> method. |
+|  [CoreStart](./kibana-plugin-server.corestart.md) | Context passed to the plugins <code>start</code> method. |
+|  [DiscoveredPlugin](./kibana-plugin-server.discoveredplugin.md) | Small container object used to expose information about discovered plugins that may or may not have been started. |
+|  [ElasticsearchError](./kibana-plugin-server.elasticsearcherror.md) |  |
+|  [ElasticsearchServiceSetup](./kibana-plugin-server.elasticsearchservicesetup.md) |  |
+|  [FakeRequest](./kibana-plugin-server.fakerequest.md) | Fake request object created manually by Kibana plugins. |
+|  [HttpServiceSetup](./kibana-plugin-server.httpservicesetup.md) |  |
+|  [HttpServiceStart](./kibana-plugin-server.httpservicestart.md) |  |
+|  [InternalCoreStart](./kibana-plugin-server.internalcorestart.md) |  |
+|  [KibanaRequestRoute](./kibana-plugin-server.kibanarequestroute.md) | Request specific route information exposed to a handler. |
+|  [Logger](./kibana-plugin-server.logger.md) | Logger exposes all the necessary methods to log any type of information and this is the interface used by the logging consumers including plugins. |
+|  [LoggerFactory](./kibana-plugin-server.loggerfactory.md) | The single purpose of <code>LoggerFactory</code> interface is to define a way to retrieve a context-based logger instance. |
+|  [LogMeta](./kibana-plugin-server.logmeta.md) | Contextual metadata |
+|  [OnPostAuthToolkit](./kibana-plugin-server.onpostauthtoolkit.md) | A tool set defining an outcome of OnPostAuth interceptor for incoming request. |
+|  [OnPreAuthToolkit](./kibana-plugin-server.onpreauthtoolkit.md) | A tool set defining an outcome of OnPreAuth interceptor for incoming request. |
+|  [Plugin](./kibana-plugin-server.plugin.md) | The interface that should be returned by a <code>PluginInitializer</code>. |
+|  [PluginInitializerContext](./kibana-plugin-server.plugininitializercontext.md) | Context that's available to plugins during initialization stage. |
+|  [PluginsServiceSetup](./kibana-plugin-server.pluginsservicesetup.md) |  |
+|  [PluginsServiceStart](./kibana-plugin-server.pluginsservicestart.md) |  |
+|  [RouteConfigOptions](./kibana-plugin-server.routeconfigoptions.md) | Route specific configuration. |
+|  [SavedObject](./kibana-plugin-server.savedobject.md) |  |
+|  [SavedObjectAttributes](./kibana-plugin-server.savedobjectattributes.md) |  |
+|  [SavedObjectReference](./kibana-plugin-server.savedobjectreference.md) | A reference to another saved object. |
+|  [SavedObjectsBaseOptions](./kibana-plugin-server.savedobjectsbaseoptions.md) |  |
+|  [SavedObjectsBulkCreateObject](./kibana-plugin-server.savedobjectsbulkcreateobject.md) |  |
+|  [SavedObjectsBulkGetObject](./kibana-plugin-server.savedobjectsbulkgetobject.md) |  |
+|  [SavedObjectsBulkResponse](./kibana-plugin-server.savedobjectsbulkresponse.md) |  |
+|  [SavedObjectsClientWrapperOptions](./kibana-plugin-server.savedobjectsclientwrapperoptions.md) |  |
+|  [SavedObjectsCreateOptions](./kibana-plugin-server.savedobjectscreateoptions.md) |  |
+|  [SavedObjectsFindOptions](./kibana-plugin-server.savedobjectsfindoptions.md) |  |
+|  [SavedObjectsFindResponse](./kibana-plugin-server.savedobjectsfindresponse.md) |  |
+|  [SavedObjectsMigrationVersion](./kibana-plugin-server.savedobjectsmigrationversion.md) | A dictionary of saved object type -<!-- -->&gt; version used to determine what migrations need to be applied to a saved object. |
+|  [SavedObjectsRawDoc](./kibana-plugin-server.savedobjectsrawdoc.md) | A raw document as represented directly in the saved object index. |
+|  [SavedObjectsService](./kibana-plugin-server.savedobjectsservice.md) |  |
+|  [SavedObjectsUpdateOptions](./kibana-plugin-server.savedobjectsupdateoptions.md) |  |
+|  [SavedObjectsUpdateResponse](./kibana-plugin-server.savedobjectsupdateresponse.md) |  |
+|  [SessionStorage](./kibana-plugin-server.sessionstorage.md) | Provides an interface to store and retrieve data across requests. |
+|  [SessionStorageFactory](./kibana-plugin-server.sessionstoragefactory.md) | SessionStorage factory to bind one to an incoming request |
+
+## Type Aliases
+
+|  Type Alias | Description |
+|  --- | --- |
+|  [APICaller](./kibana-plugin-server.apicaller.md) |  |
+|  [AuthenticationHandler](./kibana-plugin-server.authenticationhandler.md) |  |
+|  [AuthHeaders](./kibana-plugin-server.authheaders.md) | Auth Headers map |
+|  [ElasticsearchClientConfig](./kibana-plugin-server.elasticsearchclientconfig.md) |  |
+|  [GetAuthHeaders](./kibana-plugin-server.getauthheaders.md) | Get headers to authenticate a user against Elasticsearch. |
+|  [Headers](./kibana-plugin-server.headers.md) |  |
+|  [LegacyRequest](./kibana-plugin-server.legacyrequest.md) | Support Legacy platform request for the period of migration. |
+|  [OnPostAuthHandler](./kibana-plugin-server.onpostauthhandler.md) |  |
+|  [OnPreAuthHandler](./kibana-plugin-server.onpreauthhandler.md) |  |
+|  [PluginInitializer](./kibana-plugin-server.plugininitializer.md) | The <code>plugin</code> export at the root of a plugin's <code>server</code> directory should conform to this interface. |
+|  [PluginName](./kibana-plugin-server.pluginname.md) | Dedicated type for plugin name/id that is supposed to make Map/Set/Arrays that use it as a key or value more obvious. |
+|  [RecursiveReadonly](./kibana-plugin-server.recursivereadonly.md) |  |
+|  [RouteMethod](./kibana-plugin-server.routemethod.md) | The set of common HTTP methods supported by Kibana routing. |
+|  [SavedObjectsClientContract](./kibana-plugin-server.savedobjectsclientcontract.md) | \#\# SavedObjectsClient errors<!-- -->Since the SavedObjectsClient has its hands in everything we are a little paranoid about the way we present errors back to to application code. Ideally, all errors will be either:<!-- -->1. Caused by bad implementation (ie. undefined is not a function) and as such unpredictable 2. An error that has been classified and decorated appropriately by the decorators in [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md)<!-- -->Type 1 errors are inevitable, but since all expected/handle-able errors should be Type 2 the <code>isXYZError()</code> helpers exposed at <code>SavedObjectsErrorHelpers</code> should be used to understand and manage error responses from the <code>SavedObjectsClient</code>.<!-- -->Type 2 errors are decorated versions of the source error, so if the elasticsearch client threw an error it will be decorated based on its type. That means that rather than looking for <code>error.body.error.type</code> or doing substring checks on <code>error.body.error.reason</code>, just use the helpers to understand the meaning of the error:<!-- -->\`\`\`<!-- -->js if (SavedObjectsErrorHelpers.isNotFoundError(error)) { // handle 404 }<!-- -->if (SavedObjectsErrorHelpers.isNotAuthorizedError(error)) { // 401 handling should be automatic, but in case you wanted to know }<!-- -->// always rethrow the error unless you handle it throw error; \`\`\`<!-- -->\#\#\# 404s from missing index<!-- -->From the perspective of application code and APIs the SavedObjectsClient is a black box that persists objects. One of the internal details that users have no control over is that we use an elasticsearch index for persistance and that index might be missing.<!-- -->At the time of writing we are in the process of transitioning away from the operating assumption that the SavedObjects index is always available. Part of this transition is handling errors resulting from an index missing. These used to trigger a 500 error in most cases, and in others cause 404s with different error messages.<!-- -->From my (Spencer) perspective, a 404 from the SavedObjectsApi is a 404; The object the request/call was targeting could not be found. This is why \#14141 takes special care to ensure that 404 errors are generic and don't distinguish between index missing or document missing.<!-- -->\#\#\# 503s from missing index<!-- -->Unlike all other methods, create requests are supposed to succeed even when the Kibana index does not exist because it will be automatically created by elasticsearch. When that is not the case it is because Elasticsearch's <code>action.auto_create_index</code> setting prevents it from being created automatically so we throw a special 503 with the intention of informing the user that their Elasticsearch settings need to be updated.<!-- -->See [SavedObjectsErrorHelpers](./kibana-plugin-server.savedobjectserrorhelpers.md) |
+|  [SavedObjectsClientWrapperFactory](./kibana-plugin-server.savedobjectsclientwrapperfactory.md) |  |
+
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md
deleted file mode 100644
index 9892326ad58d15..00000000000000
--- a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._primary_term.md
+++ /dev/null
@@ -1,11 +0,0 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_primary\_term](./kibana-plugin-server.rawsavedobjectdoc._primary_term.md)
-
-## RawSavedObjectDoc.\_primary\_term property
-
-<b>Signature:</b>
-
-```typescript
-_primary_term?: number;
-```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md
deleted file mode 100644
index fdfac53065b677..00000000000000
--- a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._seq_no.md
+++ /dev/null
@@ -1,11 +0,0 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_seq\_no](./kibana-plugin-server.rawsavedobjectdoc._seq_no.md)
-
-## RawSavedObjectDoc.\_seq\_no property
-
-<b>Signature:</b>
-
-```typescript
-_seq_no?: number;
-```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md
deleted file mode 100644
index 17e1c144b62231..00000000000000
--- a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._source.md
+++ /dev/null
@@ -1,11 +0,0 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_source](./kibana-plugin-server.rawsavedobjectdoc._source.md)
-
-## RawSavedObjectDoc.\_source property
-
-<b>Signature:</b>
-
-```typescript
-_source: any;
-```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md b/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md
deleted file mode 100644
index aa2baf9d58e365..00000000000000
--- a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!-- Do not edit this file. It is automatically generated by API Documenter. -->
-
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md)
-
-## RawSavedObjectDoc interface
-
-A raw document as represented directly in the saved object index.
-
-<b>Signature:</b>
-
-```typescript
-export interface RawDoc 
-```
-
-## Properties
-
-|  Property | Type | Description |
-|  --- | --- | --- |
-|  [\_id](./kibana-plugin-server.rawsavedobjectdoc._id.md) | <code>string</code> |  |
-|  [\_primary\_term](./kibana-plugin-server.rawsavedobjectdoc._primary_term.md) | <code>number</code> |  |
-|  [\_seq\_no](./kibana-plugin-server.rawsavedobjectdoc._seq_no.md) | <code>number</code> |  |
-|  [\_source](./kibana-plugin-server.rawsavedobjectdoc._source.md) | <code>any</code> |  |
-|  [\_type](./kibana-plugin-server.rawsavedobjectdoc._type.md) | <code>string</code> |  |
-
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._id.md b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._id.md
similarity index 50%
rename from docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._id.md
rename to docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._id.md
index 92c088060e2a1c..cd16eadf519318 100644
--- a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._id.md
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._id.md
@@ -1,8 +1,8 @@
 <!-- Do not edit this file. It is automatically generated by API Documenter. -->
 
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_id](./kibana-plugin-server.rawsavedobjectdoc._id.md)
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsRawDoc](./kibana-plugin-server.savedobjectsrawdoc.md) &gt; [\_id](./kibana-plugin-server.savedobjectsrawdoc._id.md)
 
-## RawSavedObjectDoc.\_id property
+## SavedObjectsRawDoc.\_id property
 
 <b>Signature:</b>
 
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._primary_term.md b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._primary_term.md
new file mode 100644
index 00000000000000..c5eef82322f580
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._primary_term.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsRawDoc](./kibana-plugin-server.savedobjectsrawdoc.md) &gt; [\_primary\_term](./kibana-plugin-server.savedobjectsrawdoc._primary_term.md)
+
+## SavedObjectsRawDoc.\_primary\_term property
+
+<b>Signature:</b>
+
+```typescript
+_primary_term?: number;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._seq_no.md b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._seq_no.md
new file mode 100644
index 00000000000000..a3b9a943a708c8
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._seq_no.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsRawDoc](./kibana-plugin-server.savedobjectsrawdoc.md) &gt; [\_seq\_no](./kibana-plugin-server.savedobjectsrawdoc._seq_no.md)
+
+## SavedObjectsRawDoc.\_seq\_no property
+
+<b>Signature:</b>
+
+```typescript
+_seq_no?: number;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._source.md b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._source.md
new file mode 100644
index 00000000000000..1babaab14f14db
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._source.md
@@ -0,0 +1,11 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsRawDoc](./kibana-plugin-server.savedobjectsrawdoc.md) &gt; [\_source](./kibana-plugin-server.savedobjectsrawdoc._source.md)
+
+## SavedObjectsRawDoc.\_source property
+
+<b>Signature:</b>
+
+```typescript
+_source: any;
+```
diff --git a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._type.md b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._type.md
similarity index 50%
rename from docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._type.md
rename to docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._type.md
index d399663a1fdc11..31c40e15b53c09 100644
--- a/docs/development/core/server/kibana-plugin-server.rawsavedobjectdoc._type.md
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc._type.md
@@ -1,8 +1,8 @@
 <!-- Do not edit this file. It is automatically generated by API Documenter. -->
 
-[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [RawSavedObjectDoc](./kibana-plugin-server.rawsavedobjectdoc.md) &gt; [\_type](./kibana-plugin-server.rawsavedobjectdoc._type.md)
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsRawDoc](./kibana-plugin-server.savedobjectsrawdoc.md) &gt; [\_type](./kibana-plugin-server.savedobjectsrawdoc._type.md)
 
-## RawSavedObjectDoc.\_type property
+## SavedObjectsRawDoc.\_type property
 
 <b>Signature:</b>
 
diff --git a/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc.md b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc.md
new file mode 100644
index 00000000000000..5864a854653965
--- /dev/null
+++ b/docs/development/core/server/kibana-plugin-server.savedobjectsrawdoc.md
@@ -0,0 +1,24 @@
+<!-- Do not edit this file. It is automatically generated by API Documenter. -->
+
+[Home](./index.md) &gt; [kibana-plugin-server](./kibana-plugin-server.md) &gt; [SavedObjectsRawDoc](./kibana-plugin-server.savedobjectsrawdoc.md)
+
+## SavedObjectsRawDoc interface
+
+A raw document as represented directly in the saved object index.
+
+<b>Signature:</b>
+
+```typescript
+export interface RawDoc 
+```
+
+## Properties
+
+|  Property | Type | Description |
+|  --- | --- | --- |
+|  [\_id](./kibana-plugin-server.savedobjectsrawdoc._id.md) | <code>string</code> |  |
+|  [\_primary\_term](./kibana-plugin-server.savedobjectsrawdoc._primary_term.md) | <code>number</code> |  |
+|  [\_seq\_no](./kibana-plugin-server.savedobjectsrawdoc._seq_no.md) | <code>number</code> |  |
+|  [\_source](./kibana-plugin-server.savedobjectsrawdoc._source.md) | <code>any</code> |  |
+|  [\_type](./kibana-plugin-server.savedobjectsrawdoc._type.md) | <code>string</code> |  |
+
diff --git a/src/core/server/index.ts b/src/core/server/index.ts
index 7dbc85cf75ab06..c2e58785f07c42 100644
--- a/src/core/server/index.ts
+++ b/src/core/server/index.ts
@@ -87,7 +87,6 @@ export {
 } from './plugins';
 
 export {
-  RawSavedObjectDoc,
   SavedObject,
   SavedObjectAttributes,
   SavedObjectReference,
@@ -104,6 +103,7 @@ export {
   SavedObjectsFindOptions,
   SavedObjectsFindResponse,
   SavedObjectsMigrationVersion,
+  SavedObjectsRawDoc,
   SavedObjectsSchema,
   SavedObjectsSerializer,
   SavedObjectsService,
diff --git a/src/core/server/saved_objects/index.ts b/src/core/server/saved_objects/index.ts
index 865f8d9750570b..623c722eb95b1b 100644
--- a/src/core/server/saved_objects/index.ts
+++ b/src/core/server/saved_objects/index.ts
@@ -23,4 +23,4 @@ export { SavedObjectsSchema } from './schema';
 
 export { SavedObjectsManagement } from './management';
 
-export { SavedObjectsSerializer, RawDoc as RawSavedObjectDoc } from './serialization';
+export { SavedObjectsSerializer, RawDoc as SavedObjectsRawDoc } from './serialization';
diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md
index e609f628b0ea45..5a91462a657c79 100644
--- a/src/core/server/server.api.md
+++ b/src/core/server/server.api.md
@@ -378,22 +378,6 @@ export interface PluginsServiceStart {
     contracts: Map<PluginName, unknown>;
 }
 
-// Warning: (ae-missing-release-tag) "RawDoc" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
-// 
-// @public
-export interface RawSavedObjectDoc {
-    // (undocumented)
-    _id: string;
-    // (undocumented)
-    _primary_term?: number;
-    // (undocumented)
-    _seq_no?: number;
-    // (undocumented)
-    _source: any;
-    // (undocumented)
-    _type?: string;
-}
-
 // Warning: (ae-forgotten-export) The symbol "RecursiveReadonlyArray" needs to be exported by the entry point index.d.ts
 // 
 // @public (undocumented)
@@ -650,6 +634,22 @@ export interface SavedObjectsMigrationVersion {
     [pluginName: string]: string;
 }
 
+// Warning: (ae-missing-release-tag) "RawDoc" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
+// 
+// @public
+export interface SavedObjectsRawDoc {
+    // (undocumented)
+    _id: string;
+    // (undocumented)
+    _primary_term?: number;
+    // (undocumented)
+    _seq_no?: number;
+    // (undocumented)
+    _source: any;
+    // (undocumented)
+    _type?: string;
+}
+
 // Warning: (ae-missing-release-tag) "SavedObjectsSchema" is exported by the package, but it is missing a release tag (@alpha, @beta, @public, or @internal)
 // 
 // @public (undocumented)
@@ -670,10 +670,10 @@ export class SavedObjectsSchema {
 export class SavedObjectsSerializer {
     constructor(schema: SavedObjectsSchema);
     generateRawId(namespace: string | undefined, type: string, id?: string): string;
-    isRawSavedObject(rawDoc: RawSavedObjectDoc): any;
+    isRawSavedObject(rawDoc: SavedObjectsRawDoc): any;
     // Warning: (ae-forgotten-export) The symbol "SanitizedSavedObjectDoc" needs to be exported by the entry point index.d.ts
-    rawToSavedObject(doc: RawSavedObjectDoc): SanitizedSavedObjectDoc;
-    savedObjectToRaw(savedObj: SanitizedSavedObjectDoc): RawSavedObjectDoc;
+    rawToSavedObject(doc: SavedObjectsRawDoc): SanitizedSavedObjectDoc;
+    savedObjectToRaw(savedObj: SanitizedSavedObjectDoc): SavedObjectsRawDoc;
     }
 
 // @public (undocumented)
diff --git a/x-pack/legacy/plugins/task_manager/task_store.ts b/x-pack/legacy/plugins/task_manager/task_store.ts
index e4df0021127dc7..79ad72259c996f 100644
--- a/x-pack/legacy/plugins/task_manager/task_store.ts
+++ b/x-pack/legacy/plugins/task_manager/task_store.ts
@@ -14,7 +14,7 @@ import {
   SavedObject,
   SavedObjectAttributes,
   SavedObjectsSerializer,
-  RawSavedObjectDoc,
+  SavedObjectsRawDoc,
 } from 'src/core/server';
 import {
   ConcreteTaskInstance,
@@ -240,7 +240,7 @@ export class TaskStore {
     const rawDocs = result.hits.hits;
 
     return {
-      docs: (rawDocs as RawSavedObjectDoc[])
+      docs: (rawDocs as SavedObjectsRawDoc[])
         .map(doc => this.serializer.rawToSavedObject(doc))
         .map(doc => omit(doc, 'namespace') as SavedObject)
         .map(savedObjectToConcreteTaskInstance),