Add reindex warning for mapping type changes (#32081) (#33088)

elastic · Mar 13, 2019 · b68ba34 · b68ba34
1 parent 43352cb
commit b68ba34
Show file tree

Hide file tree

Showing 11 changed files with 123 additions and 112 deletions.
diff --git a/x-pack/plugins/upgrade_assistant/common/types.ts b/x-pack/plugins/upgrade_assistant/common/types.ts
@@ -49,11 +49,12 @@ export type ReindexSavedObject = SavedObject<ReindexOperation>;
 
 export enum ReindexWarning {
   // 6.0 -> 7.0 warnings, now unused
-  allField = 0,
-  booleanFields = 1,
+  allField,
+  booleanFields,
 
   // 7.0 -> 8.0 warnings
   apmReindex,
+  customTypeName,
 }
 
 export enum IndexGroup {

diff --git a/...ponents/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap b/...ponents/tabs/checkup/deprecations/reindex/flyout/__snapshots__/warning_step.test.tsx.snap
@@ -29,92 +29,67 @@ exports[`WarningsFlyoutStep renders 1`] = `
     <WarningCheckbox
       checkedIds={
         Object {
-          "reindexWarning-0": false,
-          "reindexWarning-1": false,
+          "reindexWarning-2": false,
+          "reindexWarning-3": false,
         }
       }
       description={
         <FormattedMessage
-          defaultMessage="The {allField} meta field is no longer supported in 7.0. Reindexing removes
-                      the {allField} field in the new index. Ensure that no application code or scripts reply on
-                      this field."
-          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningDetail"
+          defaultMessage="Mapping types are no longer supported in 8.x. This index mapping does not use the
+                    default type name, {defaultType}, and will be updated when reindexed. Ensure no application code
+                    or scripts rely on a different type."
+          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningDetail"
           values={
             Object {
-              "allField": <EuiCode>
-                _all
+              "defaultType": <EuiCode>
+                _doc
               </EuiCode>,
             }
           }
         />
       }
-      documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
+      documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html"
       label={
         <FormattedMessage
-          defaultMessage="{allField} will be removed"
-          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningTitle"
+          defaultMessage="Mapping type will be changed to {defaultType}"
+          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningTitle"
           values={
             Object {
-              "allField": <EuiCode>
-                _all
+              "defaultType": <EuiCode>
+                _doc
               </EuiCode>,
             }
           }
         />
       }
       onChange={[Function]}
-      warning={0}
+      warning={3}
     />
     <WarningCheckbox
       checkedIds={
         Object {
-          "reindexWarning-0": false,
-          "reindexWarning-1": false,
+          "reindexWarning-2": false,
+          "reindexWarning-3": false,
         }
       }
       description={
         <FormattedMessage
-          defaultMessage="If a document contain a boolean field that is neither {true} or {false}
-                   (for example, {yes}, {on}, {one}), reindexing converts these fields to {true} or {false}.
-                   Ensure that no application code or scripts rely on boolean fields in the deprecated format."
-          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningDetail"
-          values={
-            Object {
-              "false": <EuiCode>
-                false
-              </EuiCode>,
-              "on": <EuiCode>
-                "on"
-              </EuiCode>,
-              "one": <EuiCode>
-                1
-              </EuiCode>,
-              "true": <EuiCode>
-                true
-              </EuiCode>,
-              "yes": <EuiCode>
-                "yes"
-              </EuiCode>,
-            }
-          }
+          defaultMessage="Starting in version 7.0.0, APM data will be represented in the Elastic Common Schema.
+                      Historical APM data will not visible until it's reindexed."
+          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningDetail"
+          values={Object {}}
         />
       }
-      documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_field"
+      documentationUrl="https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html"
       label={
         <FormattedMessage
-          defaultMessage="Boolean data in {_source} might change"
-          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningTitle"
-          values={
-            Object {
-              "_source": <EuiCode>
-                _source
-              </EuiCode>,
-            }
-          }
+          defaultMessage="This index will be converted to ECS format"
+          id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.apmReindexWarningTitle"
+          values={Object {}}
         />
       }
       onChange={[Function]}
-      warning={1}
+      warning={2}
     />
   </EuiFlyoutBody>
   <EuiFlyoutFooter>

diff --git a/...ssistant/public/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx b/...ssistant/public/components/tabs/checkup/deprecations/reindex/flyout/warning_step.test.tsx
@@ -14,7 +14,7 @@ import { idForWarning, WarningsFlyoutStep } from './warnings_step';
 describe('WarningsFlyoutStep', () => {
   const defaultProps = {
     advanceNextStep: jest.fn(),
-    warnings: [ReindexWarning.allField, ReindexWarning.booleanFields],
+    warnings: [ReindexWarning.customTypeName, ReindexWarning.apmReindex],
     closeFlyout: jest.fn(),
   };
 
@@ -33,11 +33,11 @@ describe('WarningsFlyoutStep', () => {
     button.simulate('click');
     expect(defaultProps.advanceNextStep).not.toHaveBeenCalled();
 
-    wrapper.find(`input#${idForWarning(ReindexWarning.allField)}`).simulate('change');
+    wrapper.find(`input#${idForWarning(ReindexWarning.apmReindex)}`).simulate('change');
     button.simulate('click');
     expect(defaultProps.advanceNextStep).not.toHaveBeenCalled();
 
-    wrapper.find(`input#${idForWarning(ReindexWarning.booleanFields)}`).simulate('change');
+    wrapper.find(`input#${idForWarning(ReindexWarning.customTypeName)}`).simulate('change');
     button.simulate('click');
     expect(defaultProps.advanceNextStep).toHaveBeenCalled();
   });

diff --git a/...de_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx b/...de_assistant/public/components/tabs/checkup/deprecations/reindex/flyout/warnings_step.tsx
@@ -123,32 +123,32 @@ export class WarningsFlyoutStep extends React.Component<
 
           <EuiSpacer />
 
-          {warnings.includes(ReindexWarning.allField) && (
+          {warnings.includes(ReindexWarning.customTypeName) && (
             <WarningCheckbox
               checkedIds={checkedIds}
               onChange={this.onChange}
-              warning={ReindexWarning.allField}
+              warning={ReindexWarning.customTypeName}
               label={
                 <FormattedMessage
-                  id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningTitle"
-                  defaultMessage="{allField} will be removed"
+                  id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningTitle"
+                  defaultMessage="Mapping type will be changed to {defaultType}"
                   values={{
-                    allField: <EuiCode>_all</EuiCode>,
+                    defaultType: <EuiCode>_doc</EuiCode>,
                   }}
                 />
               }
               description={
                 <FormattedMessage
-                  id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.allFieldWarningDetail"
-                  defaultMessage="The {allField} meta field is no longer supported in 7.0. Reindexing removes
-                      the {allField} field in the new index. Ensure that no application code or scripts reply on
-                      this field."
+                  id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.customTypeNameWarningDetail"
+                  defaultMessage="Mapping types are no longer supported in 8.x. This index mapping does not use the
+                    default type name, {defaultType}, and will be updated when reindexed. Ensure no application code
+                    or scripts rely on a different type."
                   values={{
-                    allField: <EuiCode>_all</EuiCode>,
+                    defaultType: <EuiCode>_doc</EuiCode>,
                   }}
                 />
               }
-              documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_the_literal__all_literal_meta_field_is_now_disabled_by_default"
+              documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/7.0/removal-of-types.html"
             />
           )}
 
@@ -173,37 +173,6 @@ export class WarningsFlyoutStep extends React.Component<
               documentationUrl="https://www.elastic.co/guide/en/apm/get-started/master/apm-release-notes.html"
             />
           )}
-
-          {warnings.includes(ReindexWarning.booleanFields) && (
-            <WarningCheckbox
-              checkedIds={checkedIds}
-              onChange={this.onChange}
-              warning={ReindexWarning.booleanFields}
-              label={
-                <FormattedMessage
-                  id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningTitle"
-                  defaultMessage="Boolean data in {_source} might change"
-                  values={{ _source: <EuiCode>_source</EuiCode> }}
-                />
-              }
-              description={
-                <FormattedMessage
-                  id="xpack.upgradeAssistant.checkupTab.reindexing.flyout.warningsStep.booleanFieldsWarningDetail"
-                  defaultMessage="If a document contain a boolean field that is neither {true} or {false}
-                   (for example, {yes}, {on}, {one}), reindexing converts these fields to {true} or {false}.
-                   Ensure that no application code or scripts rely on boolean fields in the deprecated format."
-                  values={{
-                    true: <EuiCode>true</EuiCode>,
-                    false: <EuiCode>false</EuiCode>,
-                    yes: <EuiCode>"yes"</EuiCode>,
-                    on: <EuiCode>"on"</EuiCode>,
-                    one: <EuiCode>1</EuiCode>,
-                  }}
-                />
-              }
-              documentationUrl="https://www.elastic.co/guide/en/elasticsearch/reference/6.0/breaking_60_mappings_changes.html#_coercion_of_boolean_field"
-            />
-          )}
         </EuiFlyoutBody>
         <EuiFlyoutFooter>
           <EuiFlexGroup justifyContent="spaceBetween">

diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.test.ts
@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { ReindexWarning } from 'x-pack/plugins/upgrade_assistant/common/types';
 import {
   CURRENT_MAJOR_VERSION,
   PREV_MAJOR_VERSION,
@@ -121,4 +122,22 @@ describe('getReindexWarnings', () => {
       })
     ).toEqual([]);
   });
+
+  it('returns customTypeName for non-_doc mapping types', () => {
+    expect(
+      getReindexWarnings({
+        settings: {},
+        mappings: { doc: {} },
+      })
+    ).toEqual([ReindexWarning.customTypeName]);
+  });
+
+  it('does not return customTypeName for _doc mapping types', () => {
+    expect(
+      getReindexWarnings({
+        settings: {},
+        mappings: { _doc: {} },
+      })
+    ).toEqual([]);
+  });
 });
diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/index_settings.ts
@@ -11,7 +11,9 @@ import {
 } from 'x-pack/plugins/upgrade_assistant/common/version';
 import { ReindexWarning } from '../../../common/types';
 import { isLegacyApmIndex } from '../apm';
-import { FlatSettings } from './types';
+import { FlatSettings, FlatSettingsWithTypeName } from './types';
+
+export const DEFAULT_TYPE_NAME = '_doc';
 
 export interface ParsedIndexName {
   cleanIndexName: string;
@@ -74,19 +76,33 @@ export const generateNewIndexName = (indexName: string): string => {
  * @param flatSettings
  */
 export const getReindexWarnings = (
-  flatSettings: FlatSettings,
+  flatSettings: FlatSettingsWithTypeName,
   apmIndexPatterns: string[] = []
 ): ReindexWarning[] => {
   const indexName = flatSettings.settings['index.provided_name'];
-  const apmReindexWarning = isLegacyApmIndex(indexName, apmIndexPatterns, flatSettings.mappings);
-
-  const warnings = [[ReindexWarning.apmReindex, apmReindexWarning]] as Array<
-    [ReindexWarning, boolean]
-  >;
+  const typeName = Object.getOwnPropertyNames(flatSettings.mappings)[0];
+  const apmReindexWarning = isLegacyApmIndex(
+    indexName,
+    apmIndexPatterns,
+    flatSettings.mappings[typeName]
+  );
+  const typeNameWarning = usesCustomTypeName(flatSettings);
+
+  const warnings = [
+    [ReindexWarning.apmReindex, apmReindexWarning],
+    [ReindexWarning.customTypeName, typeNameWarning],
+  ] as Array<[ReindexWarning, boolean]>;
 
   return warnings.filter(([_, applies]) => applies).map(([warning, _]) => warning);
 };
 
+const usesCustomTypeName = (flatSettings: FlatSettingsWithTypeName) => {
+  // In 7+ it's not possible to have more than one type anyways, so always grab the first
+  // (and only) key.
+  const typeName = Object.getOwnPropertyNames(flatSettings.mappings)[0];
+  return typeName && typeName !== DEFAULT_TYPE_NAME;
+};
+
 const removeUnsettableSettings = (settings: FlatSettings['settings']) =>
   omit(settings, [
     'index.uuid',

diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_actions.ts
@@ -20,7 +20,7 @@ import {
   ReindexStep,
 } from '../../../common/types';
 import { generateNewIndexName } from './index_settings';
-import { FlatSettings } from './types';
+import { FlatSettings, FlatSettingsWithTypeName } from './types';
 
 // TODO: base on elasticsearch.requestTimeout?
 export const LOCK_WINDOW = moment.duration(90, 'seconds');
@@ -84,6 +84,12 @@ export interface ReindexActions {
    */
   getFlatSettings(indexName: string): Promise<FlatSettings | null>;
 
+  /**
+   * Retrieve index settings (in flat, dot-notation style) and mappings with the type name included.
+   * @param indexName
+   */
+  getFlatSettingsWithTypeName(indexName: string): Promise<FlatSettingsWithTypeName | null>;
+
   // ----- Functions below are for enforcing locks around groups of indices like ML or Watcher
 
   /**
@@ -244,6 +250,18 @@ export const reindexActionsFactory = (
       return flatSettings[indexName];
     },
 
+    async getFlatSettingsWithTypeName(indexName: string) {
+      const flatSettings = (await callCluster('transport.request', {
+        path: `/${encodeURIComponent(indexName)}?flat_settings=true&include_type_name=true`,
+      })) as { [indexName: string]: FlatSettingsWithTypeName };
+
+      if (!flatSettings[indexName]) {
+        return null;
+      }
+
+      return flatSettings[indexName];
+    },
+
     async _fetchAndLockIndexGroupDoc(indexGroup) {
       const fetchDoc = async () => {
         try {

diff --git a/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts b/x-pack/plugins/upgrade_assistant/server/lib/reindexing/reindex_service.test.ts
@@ -48,6 +48,7 @@ describe('reindexService', () => {
       findReindexOperations: jest.fn(unimplemented('findReindexOperations')),
       findAllByStatus: jest.fn(unimplemented('findAllInProgressOperations')),
       getFlatSettings: jest.fn(unimplemented('getFlatSettings')),
+      getFlatSettingsWithTypeName: jest.fn(unimplemented('getFlatSettingsWithTypeName')),
       cleanupChanges: jest.fn(),
       incrementIndexGroupReindexes: jest.fn(unimplemented('incrementIndexGroupReindexes')),
       decrementIndexGroupReindexes: jest.fn(unimplemented('decrementIndexGroupReindexes')),
@@ -186,12 +187,12 @@ describe('reindexService', () => {
   describe('detectReindexWarnings', () => {
     it('fetches reindex warnings from flat settings', async () => {
       const indexName = 'myIndex';
-      actions.getFlatSettings.mockResolvedValueOnce({
+      actions.getFlatSettingsWithTypeName.mockResolvedValueOnce({
         settings: {
           'index.provided_name': indexName,
         },
         mappings: {
-          properties: { https: { type: 'boolean' } },
+          _doc: { properties: { https: { type: 'boolean' } } },
         },
       });
 
@@ -200,7 +201,7 @@ describe('reindexService', () => {
     });
 
     it('returns null if index does not exist', async () => {
-      actions.getFlatSettings.mockResolvedValueOnce(null);
+      actions.getFlatSettingsWithTypeName.mockResolvedValueOnce(null);
       const reindexWarnings = await service.detectReindexWarnings('myIndex');
       expect(reindexWarnings).toBeNull();
     });