Enable failure store in discover

src/plugins/data/common/search/search_source/fetch/get_search_params.ts

@@ -42,6 +42,7 @@ export function getSearchParamsFromRequest(
     index: searchRequest.index.title || searchRequest.index,
     body,
     track_total_hits,
+    failure_store: searchRequest.failure_store,
     ...(searchRequest.index?.allowHidden && { expand_wildcards: 'all' }),
     ...searchParams,
   };

src/plugins/data/common/search/search_source/search_source.ts

@@ -780,6 +780,7 @@ export class SearchSource {
     const { getConfig } = this.dependencies;
     const searchRequest = this.mergeProps();
     searchRequest.body = searchRequest.body || {};
+    searchRequest.failure_store = this.getField('index')?.failureStoreMode || 'exclude';
     const { body, index, query, filters, highlightAll } = searchRequest;
     searchRequest.indexType = this.getIndexType(index);
     const metaFields = getConfig(UI_SETTINGS.META_FIELDS) ?? [];

src/plugins/data_view_editor/public/components/advanced_params_content/advanced_params_content.tsx

@@ -72,5 +72,20 @@ export const AdvancedParamsContent = ({
         />
       </EuiFlexItem>
     </EuiFlexGroup>
+    <EuiFlexGroup>
+      <EuiFlexItem>
+        <UseField<string, IndexPatternConfig>
+          path={'failureStoreMode'}
+          component={TextField}
+          data-test-subj="failureStoreMode"
+          componentProps={{
+            euiFieldProps: {
+              'aria-label': 'Failure store mode',
+              disabled: disableId,
+            },
+          }}
+        />
+      </EuiFlexItem>
+    </EuiFlexGroup>
   </AdvancedParamsSection>
 );

src/plugins/data_view_editor/public/components/data_view_editor_flyout_content.tsx

@@ -110,6 +110,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
             title: editData.getIndexPattern(),
             id: editData.id,
             name: editData.name,
+            failureStoreMode: editData.failureStoreMode,
             allowHidden: editData.getAllowHidden(),
             ...(editData.timeFieldName === noTimeFieldValue
               ? { timestampField: { label: noTimeFieldLabel, value: noTimeFieldValue } }
@@ -133,6 +134,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
         id: formData.id,
         name: formData.name,
         allowHidden: formData.allowHidden,
+        failureStoreMode: formData.failureStoreMode,
       };
 
       if (type === INDEX_PATTERN_TYPE.ROLLUP && rollupIndex) {
@@ -167,6 +169,7 @@ const IndexPatternEditorFlyoutContentComponent = ({
       title = schema.title.defaultValue,
       allowHidden = schema.allowHidden.defaultValue,
       type = schema.type.defaultValue,
+      failureStoreMode = schema.failureStoreMode.defaultValue,
     },
   ] = useFormData<FormInternal>({
     form,

src/plugins/data_view_editor/public/components/form_schema.ts

@@ -57,6 +57,12 @@ export const schema = {
     }),
     validations: [],
   },
+  failureStoreMode: {
+    label: 'Show failed documents',
+    helpText: 'This exposes the failure store data under the hood',
+    validations: [],
+    defaultValue: 'exclude',
+  },
   allowHidden: {
     label: i18n.translate('indexPatternEditor.editor.form.allowHiddenLabel', {
       defaultMessage: 'Allow hidden and system indices',

src/plugins/data_view_editor/public/types.ts

@@ -126,6 +126,7 @@ export interface IndexPatternConfig {
   type: INDEX_PATTERN_TYPE;
   name?: string;
   isAdHoc: boolean;
+  failureStoreMode: string;
 }
 
 export interface FormInternal extends Omit<IndexPatternConfig, 'timestampField'> {

src/plugins/data_views/common/content_management/v1/cm_services.ts

@@ -25,6 +25,7 @@ const dataViewAttributesSchema = schema.object(
     title: schema.string(),
     type: schema.maybe(schema.literal(DataViewType.ROLLUP)),
     timeFieldName: schema.maybe(schema.string()),
+    failureStoreMode: schema.maybe(schema.string()),
     sourceFilters: schema.maybe(
       schema.arrayOf(
         schema.object({

src/plugins/data_views/common/data_views/abstract_data_views.ts

@@ -122,6 +122,8 @@ export abstract class AbstractDataView {
    */
   public name: string = '';
 
+  public failureStoreMode: string | undefined;
+
   /*
    * list of indices that the index pattern matched
    */
@@ -179,6 +181,8 @@ export abstract class AbstractDataView {
 
     this.version = spec.version;
 
+    this.failureStoreMode = spec.failureStoreMode;
+
     this.title = spec.title || '';
     this.timeFieldName = spec.timeFieldName;
     this.sourceFilters = [...(spec.sourceFilters || [])];
@@ -371,6 +375,7 @@ export abstract class AbstractDataView {
       typeMeta: stringifyOrUndefined(this.typeMeta),
       allowNoIndex: this.allowNoIndex ? this.allowNoIndex : undefined,
       runtimeFieldMap: stringifyOrUndefined(this.runtimeFieldMap),
+      failureStoreMode: this.failureStoreMode,
       name: this.name,
       allowHidden: this.allowHidden,
     };

src/plugins/data_views/common/data_views/data_view.ts

@@ -74,6 +74,7 @@ export class DataView extends AbstractDataView implements DataViewBase {
 
     this.fields = fieldList([], this.shortDotsEnable);
     this.flattenHit = flattenHitWrapper(this, metaFields);
+    this.failureStoreMode = spec.failureStoreMode;
 
     // set values
     this.fields.replaceAll(Object.values(spec.fields || {}));

src/plugins/data_views/common/data_views/data_views.ts

@@ -568,6 +568,7 @@ export class DataViewsService {
     this.getFieldsForWildcard({
       type: indexPattern.type,
       rollupIndex: indexPattern?.typeMeta?.params?.rollup_index,
+      failureStore: indexPattern?.failureStoreMode,
       allowNoIndex: true,
       ...options,
       pattern: indexPattern.title as string,
@@ -586,6 +587,7 @@ export class DataViewsService {
       type: dataView.type,
       rollupIndex: dataView?.typeMeta?.params?.rollup_index,
       allowNoIndex: true,
+      failureStore: dataView.failureStoreMode,
       pattern: dataView.getIndexPattern(),
       metaFields,
       forceRefresh,
@@ -600,6 +602,7 @@ export class DataViewsService {
       metaFields,
       type: options.type,
       rollupIndex: options.rollupIndex,
+      failureStore: options.failureStore,
       allowNoIndex: true,
       indexFilter: options.indexFilter,
       allowHidden: options.allowHidden,
@@ -774,6 +777,7 @@ export class DataViewsService {
         allowNoIndex,
         name,
         allowHidden,
+        failureStoreMode,
       },
     } = savedObject;
 
@@ -792,6 +796,7 @@ export class DataViewsService {
       namespaces,
       title,
       timeFieldName,
+      failureStoreMode,
       sourceFilters: parsedSourceFilters,
       fields: this.fieldArrayToMap(parsedFields, parsedFieldAttrs),
       typeMeta: parsedTypeMeta,
@@ -823,14 +828,15 @@ export class DataViewsService {
     spec: DataViewSpec;
     displayErrors?: boolean;
   }) => {
-    const { title, type, typeMeta, runtimeFieldMap } = spec;
+    const { title, type, typeMeta, runtimeFieldMap, failureStoreMode } = spec;
     const { fields, indices, etag } = await this.refreshFieldSpecMap(
       spec.fields || {},
       savedObjectId,
       spec.title as string,
       {
         pattern: title as string,
         metaFields: await this.getMetaFields(),
+        failureStore: failureStoreMode,
         type,
         rollupIndex: typeMeta?.params?.rollup_index,
         allowNoIndex: spec.allowNoIndex,

src/plugins/data_views/common/types.ts

@@ -165,6 +165,7 @@ export interface DataViewAttributes {
    * Allow hidden and system indices when loading field list
    */
   allowHidden?: boolean;
+  failureStoreMode?: string;
 }
 
 /**
@@ -318,6 +319,7 @@ export interface GetFieldsOptions {
   rollupIndex?: string;
   allowNoIndex?: boolean;
   indexFilter?: QueryDslQueryContainer;
+  failureStore?: string;
   includeUnmapped?: boolean;
   fields?: string[];
   allowHidden?: boolean;
@@ -553,6 +555,7 @@ export type DataViewSpec = {
    * Allow hidden and system indices when loading field list
    */
   allowHidden?: boolean;
+  failureStoreMode?: string;
 };
 
 // eslint-disable-next-line @typescript-eslint/consistent-type-definitions

src/plugins/data_views/public/data_views/data_views_api_client.ts

@@ -93,6 +93,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
       indexFilter,
       includeUnmapped,
       fields,
+      failureStore,
       forceRefresh,
       allowHidden,
       fieldTypes,
@@ -110,6 +111,7 @@ export class DataViewsApiClient implements IDataViewsApiClient {
         rollup_index: rollupIndex,
         allow_no_index: allowNoIndex,
         include_unmapped: includeUnmapped,
+        failure_store: failureStore,
         fields,
         fieldTypes,
         // default to undefined to keep value out of URL params and improve caching

src/plugins/data_views/server/content_management/data_views_storage.ts

@@ -36,6 +36,7 @@ export class DataViewsStorage extends SOContentStorage<DataViewCrudTypes> {
         'fieldAttrs',
         'runtimeFieldMap',
         'allowNoIndex',
+        'failureStoreMode',
         'name',
         'allowHidden',
       ],

src/plugins/data_views/server/fetcher/index_patterns_fetcher.ts

@@ -75,6 +75,7 @@ export class IndexPatternsFetcher {
     metaFields?: string[];
     fieldCapsOptions?: { allow_no_indices: boolean; includeUnmapped?: boolean };
     type?: string;
+    failureStore?: string;
     rollupIndex?: string;
     indexFilter?: QueryDslQueryContainer;
     fields?: string[];
@@ -91,6 +92,7 @@ export class IndexPatternsFetcher {
       indexFilter,
       allowHidden,
       fieldTypes,
+      failureStore,
       includeEmptyFields,
     } = options;
     const allowNoIndices = fieldCapsOptions
@@ -109,6 +111,7 @@ export class IndexPatternsFetcher {
         include_unmapped: fieldCapsOptions?.includeUnmapped,
       },
       indexFilter,
+      failureStore,
       fields: options.fields || ['*'],
       expandWildcards,
       fieldTypes,

src/plugins/data_views/server/fetcher/lib/es_api.ts

@@ -45,6 +45,7 @@ interface FieldCapsApiParams {
   indices: string[] | string;
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
+  failureStore?: string;
   fields?: string[];
   expandWildcards?: ExpandWildcard;
   fieldTypes?: string[];
@@ -68,6 +69,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
     callCluster,
     indices,
     indexFilter,
+    failureStore,
     fieldCapsOptions = {
       allow_no_indices: false,
       include_unmapped: false,
@@ -77,6 +79,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
     fieldTypes,
     includeEmptyFields,
   } = params;
+  console.log(`XXXXX ${failureStore} XXXXXXX`);
   try {
     return await callCluster.fieldCaps(
       {
@@ -87,6 +90,7 @@ export async function callFieldCapsApi(params: FieldCapsApiParams) {
         expand_wildcards: expandWildcards,
         types: fieldTypes,
         include_empty_fields: includeEmptyFields ?? true,
+        failure_store: failureStore || 'exclude',
         ...fieldCapsOptions,
       },
       { meta: true }

src/plugins/data_views/server/fetcher/lib/field_capabilities/field_capabilities.ts

@@ -23,6 +23,7 @@ interface FieldCapabilitiesParams {
   uiSettingsClient?: IUiSettingsClient;
   indices: string | string[];
   metaFields: string[];
+  failureStore?: string;
   fieldCapsOptions?: { allow_no_indices: boolean; include_unmapped?: boolean };
   indexFilter?: QueryDslQueryContainer;
   fields?: string[];
@@ -49,6 +50,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     fieldCapsOptions,
     indexFilter,
     metaFields = [],
+    failureStore,
     fields,
     expandWildcards,
     fieldTypes,
@@ -60,6 +62,7 @@ export async function getFieldCapabilities(params: FieldCapabilitiesParams) {
     callCluster,
     indices,
     fieldCapsOptions,
+    failureStore,
     indexFilter: getIndexFilterDsl({ indexFilter, excludedTiers }),
     fields,
     expandWildcards,

src/plugins/data_views/server/index_patterns_api_client.ts

@@ -30,6 +30,7 @@ export class IndexPatternsApiServer implements IDataViewsApiClient {
     rollupIndex,
     allowNoIndex,
     indexFilter,
+    failureStore,
     fields,
   }: GetFieldsOptions) {
     const indexPatterns = new IndexPatternsFetcher(this.esClient, {
@@ -44,6 +45,7 @@ export class IndexPatternsApiServer implements IDataViewsApiClient {
         type,
         rollupIndex,
         indexFilter,
+        failureStore,
         fields,
       })
       .catch((err) => {

src/plugins/data_views/server/rest_api_routes/internal/fields.ts

@@ -41,6 +41,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
       type,
       rollup_index: rollupIndex,
       allow_no_index: allowNoIndex,
+      failure_store: failureStore,
       include_unmapped: includeUnmapped,
       field_types: fieldTypes,
     } = request.query;
@@ -62,6 +63,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
         metaFields: parsedMetaFields,
         type,
         rollupIndex,
+        failureStore,
         fieldCapsOptions: {
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,

src/plugins/data_views/server/rest_api_routes/internal/fields_for.ts

@@ -47,6 +47,7 @@ export interface IQuery {
   meta_fields: string | string[];
   type?: string;
   rollup_index?: string;
+  failure_store?: string;
   allow_no_index?: boolean;
   include_unmapped?: boolean;
   fields?: string | string[];
@@ -63,6 +64,7 @@ export const querySchema = schema.object({
   type: schema.maybe(schema.string()),
   rollup_index: schema.maybe(schema.string()),
   allow_no_index: schema.maybe(schema.boolean()),
+  failure_store: schema.maybe(schema.string()),
   include_unmapped: schema.maybe(schema.boolean()),
   fields: schema.maybe(schema.oneOf([schema.string(), schema.arrayOf(schema.string())])),
   allow_hidden: schema.maybe(schema.boolean()),
@@ -137,6 +139,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
       pattern,
       meta_fields: metaFields,
       type,
+      failure_store: failureStoreMode,
       rollup_index: rollupIndex,
       allow_no_index: allowNoIndex,
       include_unmapped: includeUnmapped,
@@ -169,6 +172,7 @@ const handler: (isRollupsEnabled: () => boolean) => RequestHandler<{}, IQuery, I
           allow_no_indices: allowNoIndex || false,
           includeUnmapped,
         },
+        failureStore: failureStoreMode,
         fieldTypes: parsedFieldTypes,
         indexFilter,
         allowHidden,