Allow creating filters from fields with null values in discover (#70936)

* Fix bug #7189

* typo

* Test adjustments

* wait for load complete

* Fine tune test

* Update src/plugins/data/public/query/filter_manager/lib/generate_filters.ts

Co-authored-by: Lukas Olson <olson.lukas@gmail.com>

* Fix filtering by an array of nulls
Allow filtering by a non existing field in the doc
simplify flatten hit logic

Co-authored-by: Lukas Olson <olson.lukas@gmail.com>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

src/plugins/data/common/index_patterns/index_patterns/flatten_hit.ts

@@ -78,7 +78,12 @@ function decorateFlattenedWrapper(hit: Record<string, any>, metaFields: Record<s
     // unwrap computed fields
     _.forOwn(hit.fields, function (val, key: any) {
       if (key[0] === '_' && !_.includes(metaFields, key)) return;
-      flattened[key] = Array.isArray(val) && val.length === 1 ? val[0] : val;
+      // Flatten an array with 0 or 1 elements to a single value.
+      if (Array.isArray(val) && val.length <= 1) {
+        flattened[key] = val[0];
+      } else {
+        flattened[key] = val;
+      }
     });
 
     return flattened;

src/plugins/data/public/query/filter_manager/lib/generate_filters.ts

@@ -106,11 +106,15 @@ export function generateFilters(
       // exists filter special case:  fieldname = '_exists' and value = fieldname
       const filterType = fieldName === '_exists_' ? FILTERS.EXISTS : FILTERS.PHRASE;
       const actualFieldObj = fieldName === '_exists_' ? ({ name: value } as IFieldType) : fieldObj;
+
+      // Fix for #7189 - if value is empty, phrase filters become exists filters.
+      const isNullFilter = value === null || value === undefined;
+
       filter = buildFilter(
         tmpIndexPattern,
         actualFieldObj,
-        filterType,
-        negate,
+        isNullFilter ? FILTERS.EXISTS : filterType,
+        isNullFilter ? !negate : negate,
         false,
         value,
         null,

src/plugins/data/public/ui/filter_bar/filter_item.tsx

@@ -135,9 +135,10 @@ export function FilterItem(props: Props) {
     const dataTestSubjValue = filter.meta.value
       ? `filter-value-${isValidLabel(labelConfig) ? labelConfig.title : labelConfig.status}`
       : '';
+    const dataTestSubjNegated = filter.meta.negate ? 'filter-negated' : '';
     const dataTestSubjDisabled = `filter-${isDisabled(labelConfig) ? 'disabled' : 'enabled'}`;
     const dataTestSubjPinned = `filter-${isFilterPinned(filter) ? 'pinned' : 'unpinned'}`;
-    return `filter ${dataTestSubjDisabled} ${dataTestSubjKey} ${dataTestSubjValue} ${dataTestSubjPinned}`;
+    return `filter ${dataTestSubjDisabled} ${dataTestSubjKey} ${dataTestSubjValue} ${dataTestSubjPinned} ${dataTestSubjNegated}`;
   }
 
   function getPanels() {

src/plugins/discover/public/application/angular/doc_table/components/table_row.ts

@@ -151,11 +151,7 @@ export function createTableRowDirective($compile: ng.ICompileService, $httpParam
         }
 
         $scope.columns.forEach(function (column: any) {
-          const isFilterable =
-            $scope.flattenedRow[column] !== undefined &&
-            mapping(column) &&
-            mapping(column).filterable &&
-            $scope.filter;
+          const isFilterable = mapping(column) && mapping(column).filterable && $scope.filter;
 
           newHtmls.push(
             cellTemplate({

test/functional/apps/discover/_doc_navigation.js

@@ -20,14 +20,19 @@
 import expect from '@kbn/expect';
 
 export default function ({ getService, getPageObjects }) {
+  const log = getService('log');
   const docTable = getService('docTable');
+  const filterBar = getService('filterBar');
   const testSubjects = getService('testSubjects');
-  const PageObjects = getPageObjects(['common', 'discover', 'timePicker']);
+  const PageObjects = getPageObjects(['common', 'discover', 'timePicker', 'context']);
   const esArchiver = getService('esArchiver');
   const retry = getService('retry');
 
   describe('doc link in discover', function contextSize() {
-    before(async function () {
+    beforeEach(async function () {
+      log.debug('load kibana index with default index pattern');
+      await esArchiver.loadIfNeeded('discover');
+
       await esArchiver.loadIfNeeded('logstash_functional');
       await PageObjects.common.navigateToApp('discover');
       await PageObjects.timePicker.setDefaultAbsoluteRange();
@@ -50,5 +55,27 @@ export default function ({ getService, getPageObjects }) {
       const hasDocHit = await testSubjects.exists('doc-hit');
       expect(hasDocHit).to.be(true);
     });
+
+    it('add filter should create an exists filter if value is null (#7189)', async function () {
+      await PageObjects.discover.waitUntilSearchingHasFinished();
+      // Filter special document
+      await filterBar.addFilter('agent', 'is', 'Missing/Fields');
+      await PageObjects.discover.waitUntilSearchingHasFinished();
+
+      // navigate to the doc view
+      await docTable.clickRowToggle({ rowIndex: 0 });
+
+      const details = await docTable.getDetailsRow();
+      await docTable.addInclusiveFilter(details, 'referer');
+      await PageObjects.discover.waitUntilSearchingHasFinished();
+
+      const hasInclusiveFilter = await filterBar.hasFilter('referer', 'exists', true, false, true);
+      expect(hasInclusiveFilter).to.be(true);
+
+      await docTable.removeInclusiveFilter(details, 'referer');
+      await PageObjects.discover.waitUntilSearchingHasFinished();
+      const hasExcludeFilter = await filterBar.hasFilter('referer', 'exists', true, false, false);
+      expect(hasExcludeFilter).to.be(true);
+    });
   });
 }

test/functional/services/doc_table.ts

@@ -58,6 +58,11 @@ export function DocTableProvider({ getService, getPageObjects }: FtrProviderCont
         : (await this.getBodyRows())[options.rowIndex];
     }
 
+    public async getDetailsRow(): Promise<WebElementWrapper> {
+      const table = await this.getTable();
+      return await table.findByCssSelector('[data-test-subj~="docTableDetailsRow"]');
+    }
+
     public async getAnchorDetailsRow(): Promise<WebElementWrapper> {
       const table = await this.getTable();
       return await table.findByCssSelector(
@@ -133,6 +138,22 @@ export function DocTableProvider({ getService, getPageObjects }: FtrProviderCont
       await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
     }
 
+    public async getRemoveInclusiveFilterButton(
+      tableDocViewRow: WebElementWrapper
+    ): Promise<WebElementWrapper> {
+      return await tableDocViewRow.findByTestSubject(`~removeInclusiveFilterButton`);
+    }
+
+    public async removeInclusiveFilter(
+      detailsRow: WebElementWrapper,
+      fieldName: WebElementWrapper
+    ): Promise<void> {
+      const tableDocViewRow = await this.getTableDocViewRow(detailsRow, fieldName);
+      const addInclusiveFilterButton = await this.getRemoveInclusiveFilterButton(tableDocViewRow);
+      await addInclusiveFilterButton.click();
+      await PageObjects.header.awaitGlobalLoadingIndicatorHidden();
+    }
+
     public async getAddExistsFilterButton(
       tableDocViewRow: WebElementWrapper
     ): Promise<WebElementWrapper> {

test/functional/services/filter_bar.ts

@@ -31,17 +31,21 @@ export function FilterBarProvider({ getService, getPageObjects }: FtrProviderCon
      * @param key field name
      * @param value filter value
      * @param enabled filter status
+     * @param pinned filter pinned status
+     * @param negated filter including or excluding value
      */
     public async hasFilter(
       key: string,
       value: string,
       enabled: boolean = true,
-      pinned: boolean = false
+      pinned: boolean = false,
+      negated: boolean = false
     ): Promise<boolean> {
       const filterActivationState = enabled ? 'enabled' : 'disabled';
       const filterPinnedState = pinned ? 'pinned' : 'unpinned';
+      const filterNegatedState = negated ? 'filter-negated' : '';
       return testSubjects.exists(
-        `filter filter-${filterActivationState} filter-key-${key} filter-value-${value} filter-${filterPinnedState}`,
+        `filter filter-${filterActivationState} filter-key-${key} filter-value-${value} filter-${filterPinnedState} ${filterNegatedState}`,
         {
           allowHidden: true,
         }