Support for reindexing APM indices

src/legacy/core_plugins/apm_oss/index.d.ts

@@ -0,0 +1,22 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export interface ApmOssPlugin {
+  indexPatterns: string[];
+}

src/legacy/core_plugins/apm_oss/index.js

@@ -17,6 +17,8 @@
  * under the License.
  */
 
+import _ from 'lodash';
+
 export default function apmOss(kibana) {
   return new kibana.Plugin({
     id: 'apm_oss',
@@ -30,12 +32,24 @@ export default function apmOss(kibana) {
         indexPattern: Joi.string().default('apm-*'),
 
         // ES Indices
+        sourcemapIndices: Joi.string().default('apm-*'),
         errorIndices: Joi.string().default('apm-*'),
-        onboardingIndices: Joi.string().default('apm-*'),
-        spanIndices: Joi.string().default('apm-*'),
         transactionIndices: Joi.string().default('apm-*'),
+        spanIndices: Joi.string().default('apm-*'),
         metricsIndices: Joi.string().default('apm-*'),
+        onboardingIndices: Joi.string().default('apm-*'),
       }).default();
     },
+
+    init(server) {
+      server.expose('indexPatterns', _.uniq([
+        'sourcemapIndices',
+        'errorIndices',
+        'transactionIndices',
+        'spanIndices',
+        'metricsIndices',
+        'onboardingIndices'
+      ].map(type => server.config().get(`apm_oss.${type}`))));
+    }
   });
 }

src/legacy/core_plugins/elasticsearch/index.d.ts

@@ -202,12 +202,14 @@ export interface DeprecationInfo {
   details?: string;
 }
 
+export interface IndexSettingsDeprecationInfo {
+  [indexName: string]: DeprecationInfo[];
+}
+
 export interface DeprecationAPIResponse {
   cluster_settings: DeprecationInfo[];
   node_settings: DeprecationInfo[];
-  index_settings: {
-    [indexName: string]: DeprecationInfo[];
-  };
+  index_settings: IndexSettingsDeprecationInfo;
 }
 
 export interface CallClusterOptions {

src/server/kbn_server.d.ts

@@ -19,7 +19,9 @@
 
 import { Server } from 'hapi';
 
+import { ApmOssPlugin } from '../legacy/core_plugins/apm_oss';
 import { CallClusterWithRequest, ElasticsearchPlugin } from '../legacy/core_plugins/elasticsearch';
+
 import { IndexPatternsServiceFactory } from './index_patterns';
 import { SavedObjectsClient, SavedObjectsService } from './saved_objects';
 
@@ -33,6 +35,7 @@ declare module 'hapi' {
     elasticsearch: ElasticsearchPlugin;
     kibana: any;
     spaces: any;
+    apm_oss: ApmOssPlugin;
     // add new plugin types here
   }
 

x-pack/plugins/upgrade_assistant/common/types.ts

@@ -49,4 +49,5 @@ export enum ReindexWarning {
   booleanFields = 1,
 
   // 7.0 -> 8.0 warnings
+  apmReindex,
 }

x-pack/plugins/upgrade_assistant/index.ts

@@ -3,8 +3,8 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { Server } from 'hapi';
 import Joi from 'joi';
+import { Legacy } from 'kibana';
 import { resolve } from 'path';
 import { initServer } from './server';
 
@@ -30,7 +30,7 @@ export function upgradeAssistant(kibana: any) {
       }).default();
     },
 
-    init(server: Server) {
+    init(server: Legacy.Server) {
       // Add server routes and initialize the plugin here
       initServer(server);
     },

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/index_table.tsx

@@ -135,7 +135,7 @@ export class IndexDeprecationTableUI extends React.Component<
     // NOTE: this naive implementation assumes all indices in the table are
     // should show the reindex button. This should work for known usecases.
     const { indices } = this.props;
-    if (!indices.find(i => i.reindex)) {
+    if (!indices.find(i => i.reindex === true)) {
       return null;
     }
 

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/list.tsx

@@ -10,13 +10,10 @@ import { DeprecationInfo } from 'src/legacy/core_plugins/elasticsearch';
 import { EnrichedDeprecationInfo } from '../../../../../server/lib/es_migration_apis';
 import { GroupByOption } from '../../../types';
 
-import { CURRENT_MAJOR_VERSION } from 'x-pack/plugins/upgrade_assistant/common/version';
 import { COLOR_MAP, LEVEL_MAP } from '../constants';
 import { DeprecationCell } from './cell';
 import { IndexDeprecationDetails, IndexDeprecationTable } from './index_table';
 
-const OLD_INDEX_MESSAGE = `Index created before ${CURRENT_MAJOR_VERSION}.0`;
-
 const sortByLevelDesc = (a: DeprecationInfo, b: DeprecationInfo) => {
   return -1 * (LEVEL_MAP[a.level] - LEVEL_MAP[b.level]);
 };
@@ -37,7 +34,7 @@ const MessageDeprecation: StatelessComponent<{ deprecation: EnrichedDeprecationI
     <DeprecationCell
       headline={deprecation.message}
       healthColor={COLOR_MAP[deprecation.level]}
-      reindexIndexName={deprecation.message === OLD_INDEX_MESSAGE ? deprecation.index! : undefined}
+      reindexIndexName={deprecation.reindex ? deprecation.index! : undefined}
       docUrl={deprecation.url}
       items={items}
     />
@@ -91,7 +88,7 @@ export const DeprecationList: StatelessComponent<{
     const indices = deprecations.map(dep => ({
       index: dep.index!,
       details: dep.details,
-      reindex: dep.message === OLD_INDEX_MESSAGE,
+      reindex: dep.reindex,
     }));
 
     return <IndexDeprecation indices={indices} deprecation={deprecations[0]} />;

x-pack/plugins/upgrade_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx

@@ -107,6 +107,35 @@ export class WarningsFlyoutStep extends React.Component<
 
           <EuiSpacer />
 
+          {warnings.includes(ReindexWarning.apmReindex) && (
+            <EuiText>
+              <EuiCheckbox
+                id={idForWarning(ReindexWarning.apmReindex)}
+                label={<strong>This index will be converted to ECS format</strong>}
+                checked={checkedIds[idForWarning(ReindexWarning.apmReindex)]}
+                onChange={this.onChange}
+              />
+              <p className="upgWarningsStep__warningDescription">
+                Starting in version 7.0.0, APM data will be represented in the Elastic Common
+                Schema. In order for legacy data to be included, it is required that the data is
+                re-indexed to support this new format.
+                <br />
+                <EuiLink
+                  href="https://www.elastic.co/guide/en/apm/server/master/breaking-changes.html"
+                  target="_blank"
+                >
+                  Documentation
+                </EuiLink>
+                <br />
+                <EuiLink href="https://github.com/elastic/ecs" target="_blank">
+                  More about ECS
+                </EuiLink>
+              </p>
+            </EuiText>
+          )}
+
+          <EuiSpacer />
+
           {warnings.includes(ReindexWarning.booleanFields) && (
             <EuiText>
               <EuiCheckbox

x-pack/plugins/upgrade_assistant/server/lib/apm/index.test.ts

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { getDeprecatedApmIndices, isLegacyApmIndex } from './';
+
+function mockedCallWithRequest() {
+  return jest.fn().mockImplementation(async () => {
+    return {
+      'foo-1': {
+        mappings: {},
+      },
+      'foo-2': {
+        mappings: {
+          _meta: {
+            version: '6.7.0',
+          },
+        },
+      },
+      'foo-3': {
+        mappings: {
+          _meta: {
+            version: '7.0.0',
+          },
+        },
+      },
+      'foo-4': {
+        mappings: {
+          _meta: {
+            version: '7.1.0',
+          },
+        },
+      },
+    };
+  });
+}
+
+describe('getDeprecatedApmIndices', () => {
+  it('calls indices.getMapping', async () => {
+    const callWithRequest = mockedCallWithRequest();
+    await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*', 'bar-*']);
+
+    expect(callWithRequest).toHaveBeenCalledWith({}, 'indices.getMapping', {
+      index: 'foo-*,bar-*',
+      filterPath: '*.mappings._meta.version,*.mappings.properties.@timestamp',
+    });
+  });
+
+  it('includes mappings not yet at 7.0.0', async () => {
+    const callWithRequest = mockedCallWithRequest();
+    const deprecations = await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*']);
+
+    expect(deprecations).toHaveLength(2);
+    expect(deprecations[0].index).toEqual('foo-1');
+    expect(deprecations[1].index).toEqual('foo-2');
+  });
+
+  it('formats the deprecations', async () => {
+    const callWithRequest = mockedCallWithRequest();
+    // @ts-ignore
+    const [deprecation, _] = await getDeprecatedApmIndices(callWithRequest, {} as any, ['foo-*']);
+
+    expect(deprecation.level).toEqual('critical');
+    expect(deprecation.message).toEqual('APM index needs converted to ECS format');
+    expect(deprecation.url).toEqual(
+      'https://www.elastic.co/guide/en/apm/server/master/breaking-changes.html'
+    );
+    expect(deprecation.details).toEqual('This index was created prior to 7.0.');
+    expect(deprecation.reindex).toBe(true);
+  });
+});
+
+describe.only('isLegacyApmIndex', () => {
+  it('is true when for no version', () => {
+    expect(isLegacyApmIndex('foo-1', ['foo-*'], {})).toEqual(true);
+  });
+
+  it('is true when version is less than 7.0.0', () => {
+    expect(
+      isLegacyApmIndex('foo-1', ['foo-*'], {
+        _meta: { version: '6.7.0' },
+      })
+    ).toEqual(true);
+  });
+
+  it('is false when version is 7.0.0', () => {
+    expect(
+      isLegacyApmIndex('foo-1', ['foo-*'], {
+        _meta: { version: '7.0.0' },
+      })
+    ).toEqual(false);
+  });
+
+  it('is false when version is greater than 7.0.0', () => {
+    expect(
+      isLegacyApmIndex('foo-1', ['foo-*'], {
+        _meta: { version: '7.1.0' },
+      })
+    ).toEqual(false);
+  });
+
+  it('handles multiple index patterns', () => {
+    expect(
+      isLegacyApmIndex('bar-1', ['foo-*', 'bar-*'], {
+        _meta: { version: '6.7.0' },
+      })
+    ).toEqual(true);
+
+    expect(
+      isLegacyApmIndex('bar-1', ['foo-*', 'bar-*'], {
+        _meta: { version: '7.0.0' },
+      })
+    ).toEqual(false);
+  });
+});