[backend] build resolution map before isStixMatchFilterGroup

opencti-platform/opencti-graphql/src/utils/filtering.js

@@ -26,7 +26,9 @@ export const RELATION_FROM = 'fromId';
 export const RELATION_TO = 'toId';
 export const INSTANCE_FILTER = 'elementId';
 export const NEGATION_FILTER_SUFFIX = '_not_eq';
+export const LABEL_FILTER = 'labelledBy';
 export const RESOLUTION_FILTERS = [
+  LABEL_FILTER,
   MARKING_FILTER,
   CREATED_BY_FILTER,
   ASSIGNEE_FILTER,
@@ -44,7 +46,6 @@ export const ENTITY_FILTERS = [
   OBJECT_CONTAINS_FILTER,
 ];
 // Values
-export const LABEL_FILTER = 'labelledBy';
 export const TYPE_FILTER = 'entity_type';
 export const INDICATOR_FILTER = 'indicator_types';
 export const SCORE_FILTER = 'x_opencti_score';

opencti-platform/opencti-graphql/src/utils/stix-filtering/stix-filtering.ts

@@ -1,88 +1,152 @@
-import { INDICATOR_FILTER, } from '../filtering';
+import {
+  CREATED_BY_FILTER,
+  INDICATOR_FILTER, INSTANCE_FILTER,
+  PARTICIPANT_FILTER, RELATION_FROM, RELATION_TO,
+} from '../filtering';
 import { FILTER_KEY_TESTERS_MAP } from './stix-testers';
 import { testFilterGroup } from './boolean-logic-engine';
 import type { Filter, FilterGroup } from './filter-group';
-import { getEntitiesMapFromCache } from '../../database/cache';
 import { isUserCanAccessStixElement, SYSTEM_USER } from '../access';
-import { ENTITY_TYPE_RESOLVED_FILTERS } from '../../schema/stixDomainObject';
 import type { AuthContext, AuthUser } from '../../types/user';
-import type { StixId, StixObject } from '../../types/stix-common';
-import { extractStixRepresentative } from '../../database/stix-representative';
+import { getEntitiesMapFromCache } from '../../database/cache';
+import type { StixObject } from '../../types/stix-common';
+import { ENTITY_TYPE_RESOLVED_FILTERS } from '../../schema/stixDomainObject';
+import { STIX_EXT_OCTI } from '../../types/stix-extensions';
 
-// TODO: changed by Cathia for #2686, to integrate properly next
+// TODO: changed by Cathia, to integrate properly with her
+const ASSIGNEE_FILTER = 'objectAssignee';
 const LABEL_FILTER = 'objectLabel';
+const MARKING_FILTER = 'objectMarking';
+const OBJECT_CONTAINS_FILTER = 'objects';
+
+//----------------------------------------------------------------------------------------------------------------------
+
+/**
+ * Pass through all individual filters and throws an error if it cannot be handled properly.
+ * This is very aggressive but will allow us to detect rapidly any corner-case.
+ */
+export const validateFilter = (filter: Filter) => {
+  if (filter.key.length !== 1) {
+    throw new Error(`Stix filtering can only be executed on a unique filter key - got ${JSON.stringify(filter.key)}`);
+  }
+  if (FILTER_KEY_TESTERS_MAP[filter.key[0]] === undefined) {
+    throw new Error(`Stix filtering is not compatible with the provided filter key ${JSON.stringify(filter.key)}`);
+  }
+};
 
-//----------------------------------------------------------------------------------------------------------
+/**
+ * Recursively call validateFilter inside a FilterGroup
+ */
+export const validateFilterGroup = (filterGroup: FilterGroup) => {
+  filterGroup.filters.forEach((f) => validateFilter(f));
+  filterGroup.filterGroups.forEach((fg) => validateFilterGroup(fg));
+};
 
-type ResolutionMap = Map<string | StixId, StixObject>;
+//----------------------------------------------------------------------------------------------------------------------
 
 /**
- * Resolve some of the values (recursively) inside the filter group
- * so that testers work properly on either the unresolved ids or the resolved values (from the ids)
- * To date, we need to use the resolved values instead of ids for: Indicators, labels.
+ * Resolve some of the filter values according to a resolution map.
+ * This concerns attributes that are not directly compared with a stix attribute due to modelization differences.
+ * For instance, labels are entities internally, and filter.values would contain these entities ids.
+ * In Stix, the labels are stored in plain text: we need to replace the ids in filter.values with their resolution.
  */
-export const resolveFilter = (filter: Filter, resolutionMap: ResolutionMap): Filter => {
-  // resolve labels and indicators values using the resolutionMap
-  if (filter.key[0] === INDICATOR_FILTER || filter.key[0] === LABEL_FILTER) {
-    const newFilterValues: string [] = [];
-    filter.values.forEach((id) => {
-      const resolution = resolutionMap.get(id);
-      if (resolution) {
-        const value = extractStixRepresentative(resolution);
-        newFilterValues.push(value);
-      }
-    });
+export const resolveFilter = (filter: Filter, resolutionMap: FilterResolutionMap): Filter => {
+  const newFilterValues: string [] = [];
+  filter.values.forEach((v) => {
+    const resolution = resolutionMap.get(v);
+    if (resolution) {
+      newFilterValues.push(resolution);
+    } else {
+      newFilterValues.push(v);
+    }
+  });
 
-    return {
-      ...filter,
-      values: newFilterValues
-    };
-  }
-  // filter is untouched otherwise
-  return filter;
+  return {
+    ...filter,
+    values: newFilterValues
+  };
 };
 
 /**
- * Recursively resolve some ids in the filter, see resolveFilter
- * @param filterGroup
- * @param resolutionMap the map <id, StixObject> holding the whole object resolution
+ * Recursively call resolveFilter inside a filter group
  */
-export const resolveFilterGroup = (filterGroup: FilterGroup, resolutionMap: ResolutionMap): FilterGroup => {
+export const resolveFilterGroup = (filterGroup: FilterGroup, resolutionMap: FilterResolutionMap): FilterGroup => {
   return {
     ...filterGroup,
     filters: filterGroup.filters.map((f) => resolveFilter(f, resolutionMap)),
     filterGroups: filterGroup.filterGroups.map((fg) => resolveFilterGroup(fg, resolutionMap))
   };
 };
 
+//----------------------------------------------------------------------------------------------------------------------
+
+type FilterResolutionMap = Map<string, string>;
+
+// We use '.' for compound paths
+const RESOLUTION_MAP_PATHS: Record<string, string> = {
+  [ASSIGNEE_FILTER]: 'id', // assignee --> resolve with the standard id (which is the stix.id)
+  [CREATED_BY_FILTER]: 'id', // created by --> resolve with the standard id (which is the stix.id)
+  [LABEL_FILTER]: 'value', // labels --> resolve id to stix.name
+  [INDICATOR_FILTER]: 'type', // indicator types --> resolve id to stix.type
+  [INSTANCE_FILTER]: `extensions.${STIX_EXT_OCTI}.id`, // element id --> resolve to the id in octi extension
+  [MARKING_FILTER]: 'id', // marking --> resolve id to standard id (which is the stix.id)
+  [OBJECT_CONTAINS_FILTER]: 'id',
+  [PARTICIPANT_FILTER]: 'id', // participant --> resolve with the standard id (which is the stix.id)
+  [RELATION_FROM]: 'id',
+  [RELATION_TO]: 'id',
+};
+
+// utility function that works like R.path, giving the content of the field at "path" inside "obj"
+const getFieldByPath = (obj: { [key: string]: any; }, path: string[]): any | undefined => {
+  return path.reduce((acc, key) => (acc && acc[key] !== undefined ? acc[key] : undefined), obj);
+};
+
 /**
- * Pass through all individual filters and throws an error if it cannot be handled properly.
- * This is very aggressive but will allow us to detect rapidly any corner-case.
+ * Build a resolution map thanks to the cache
+ * @param mutableMap
+ * @param filter
+ * @param cache
  */
-export const validateFilter = (filter: Filter) => {
-  if (filter.key.length !== 1) {
-    throw new Error(`Stix filtering can only be executed on a unique filter key - got ${JSON.stringify(filter.key)}`);
-  }
-  if (FILTER_KEY_TESTERS_MAP[filter.key[0]] === undefined) {
-    throw new Error(`Stix filtering is not compatible with the provided filter key ${JSON.stringify(filter.key)}`);
+const buildResolutionMapForFilter = async (mutableMap: FilterResolutionMap, filter: Filter, cache: Map<string, StixObject>) => {
+  if (Object.keys(RESOLUTION_MAP_PATHS).includes(filter.key[0])) {
+    filter.values.forEach((v) => {
+      // manipulating proper stix objects typing requires a lot of refactoring at this point (typeguards, etc)
+      // like with isStixMatchFilterGroup, let's use any to describe our stix objects in cache
+      const cachedObject = cache.get(v) as any;
+      const path = RESOLUTION_MAP_PATHS[filter.key[0]];
+      if (cachedObject && path) {
+        const cachedValue = getFieldByPath(cachedObject, path.split('.'));
+        if (typeof cachedValue === 'string') {
+          mutableMap.set(v, cachedValue);
+        }
+      }
+    });
   }
 };
 
 /**
- * Recursively call validateFilter inside a FilterGroup
+ * recursively call buildResolutionMapForFilter inside a filter group
  */
-export const validateFilterGroup = (filterGroup: FilterGroup) => {
-  filterGroup.filters.forEach((f) => validateFilter(f));
-  filterGroup.filterGroups.forEach((fg) => validateFilterGroup(fg));
+const buildResolutionMapForFilterGroup = async (mutableMap: FilterResolutionMap, filterGroup: FilterGroup, cache: Map<string, StixObject>) => {
+  filterGroup.filters.forEach((f) => buildResolutionMapForFilter(mutableMap, f, cache));
+  filterGroup.filterGroups.forEach((fg) => buildResolutionMapForFilterGroup(mutableMap, fg, cache));
 };
 
+//----------------------------------------------------------------------------------------------------------------------
+
 /**
- * Tells if a stix object matches a given FilterGroup.
- * This function returns false when user does not have sufficient access rights on the object.
- * @throws {Error} when filter group is invalid (keys not handled).
+ * Middleware function that allow us to make unit tests by mocking the resolution map.
+ * This is necessary because the map is built thanks to the cache, not available in unit tests.
  */
-export const isStixMatchFilterGroup = async (context: AuthContext, user: AuthUser, stix: any, filterGroup: FilterGroup) : Promise<boolean> => {
+export const isStixMatchFilterGroup_MockableForUnitTests = async (
+  context: AuthContext,
+  user: AuthUser,
+  stix: any,
+  filterGroup: FilterGroup,
+  resolutionMap: FilterResolutionMap
+) : Promise<boolean> => {
   // throws on unhandled filter groups
+  // this is a failsafe, but if a valid use-case throws error here, consider adding a missing tester.
   validateFilterGroup(filterGroup);
 
   // first check: user access right (according to markings, organization, etc.)
@@ -91,15 +155,33 @@ export const isStixMatchFilterGroup = async (context: AuthContext, user: AuthUse
     return false;
   }
 
-  // TODO add the preprocessing done by Cathia for #2686
-  // that enhance the ids with standard ids, check access rights of some ids etc.
-
-  // get the resolution map once and for all, not in the recursion
-  const resolutionMap: ResolutionMap = await getEntitiesMapFromCache<StixObject>(context, SYSTEM_USER, ENTITY_TYPE_RESOLVED_FILTERS);
-
-  //  resolve some of the ids as we filter on their corresponding values
   const resolvedFilterGroup = resolveFilterGroup(filterGroup, resolutionMap);
 
   // then call our boolean engine on the filter group using the stix testers
   return testFilterGroup(stix, resolvedFilterGroup, FILTER_KEY_TESTERS_MAP);
 };
+
+/**
+ * Tells if a stix object matches a filter group given a certain context.
+ * The input filter group is a stored filter (streams, triggers, playbooks), the stix object comes from the raw stream.
+ *
+ * This function will first check the user access rights to the stix object, then resolve parts of the filter groups if necessary,
+ * prior to actually comparing the filter values with the stix values.
+ * @param context
+ * @param user
+ * @param stix stix object from the raw event stream
+ * @param filterGroup
+ * @throws {Error} on invalid filter keys
+ */
+export const isStixMatchFilterGroup = async (context: AuthContext, user: AuthUser, stix: any, filterGroup: FilterGroup) : Promise<boolean> => {
+  // resolve some of the ids as we filter on their corresponding values or standard-id for instance
+  // the provided map contains replacements for filter values, if any.
+  const map = new Map<string, string>();
+
+  // we use the entities stored in cache for the "Resolved-Filters" (all the entities used by the saved filters - stream, trigger, playbooks)
+  // see cacheManager.ts:platformResolvedFilters
+  const cache = await getEntitiesMapFromCache<StixObject>(context, SYSTEM_USER, ENTITY_TYPE_RESOLVED_FILTERS);
+  await buildResolutionMapForFilterGroup(map, filterGroup, cache);
+
+  return isStixMatchFilterGroup_MockableForUnitTests(context, user, stix, filterGroup, map);
+};

opencti-platform/opencti-graphql/src/utils/stix-filtering/stix-testers.ts

@@ -81,7 +81,7 @@ export const testEntityType = (stix: any, filter: Filter) => {
  * INDICATORS
  * - search must be insensitive to case due to constraint in frontend keywords (using "runtimeAttribute" based on keyword which is always lowercase)
  */
-export const testIndicator = (stix: any, filter: Filter) => {
+export const testIndicatorTypes = (stix: any, filter: Filter) => {
   const stixValues: string[] = stix.indicator_types ?? [];
   return testStringFilter(filter, stixValues);
 };
@@ -339,7 +339,7 @@ export const FILTER_KEY_TESTERS_MAP: Record<string, TesterFunction> = {
   [CREATED_BY_FILTER]: testCreatedBy,
   [CREATOR_FILTER]: testCreator,
   [DETECTION_FILTER]: testDetection,
-  [INDICATOR_FILTER]: testIndicator,
+  [INDICATOR_FILTER]: testIndicatorTypes,
   [LABEL_FILTER]: testLabel,
   [MAIN_OBSERVABLE_TYPE_FILTER]: testMainObservableType,
   [MARKING_FILTER]: testMarkingFilter,

opencti-platform/opencti-graphql/tests/02-integration/01-database/stix-filtering-test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from 'vitest';
 
 import { ADMIN_USER, buildStandardUser, testContext } from '../../utils/testQuery';
-import { isStixMatchFilterGroup } from '../../../src/utils/stix-filtering/stix-filtering';
+import { isStixMatchFilterGroup_MockableForUnitTests } from '../../../src/utils/stix-filtering/stix-filtering';
 import type { FilterGroup } from '../../../src/utils/stix-filtering/filter-group';
 
 import stixReports from '../../data/stream-events/stream-event-stix2-reports.json';
@@ -13,6 +13,10 @@ const stixIndicator = stixIndicators[0]; // confidence 75, revoked=true, no labe
 const TLP_CLEAR_ID = 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9';
 const WHITE_TLP = { standard_id: TLP_CLEAR_ID, internal_id: '' };
 
+const MOCK_RESOLUTION_MAP: Map<string, string> = new Map();
+MOCK_RESOLUTION_MAP.set('id-for-label-indicator', 'indicator');
+MOCK_RESOLUTION_MAP.set('id-for-marking-tlp:green', 'marking-definition--613f2e26-407d-48c7-9eca-b8e91df99dc9');
+
 describe('Stix Filtering', () => {
   it('matches stix objects with basic filter groups', async () => {
     let filterGroup: FilterGroup = {
@@ -25,8 +29,8 @@ describe('Stix Filtering', () => {
       }],
       filterGroups: [],
     };
-    expect(await isStixMatchFilterGroup(testContext, ADMIN_USER, stixReport, filterGroup)).toEqual(true);
-    expect(await isStixMatchFilterGroup(testContext, ADMIN_USER, stixIndicator, filterGroup)).toEqual(false);
+    expect(await isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixReport, filterGroup, MOCK_RESOLUTION_MAP)).toEqual(true);
+    expect(await isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixIndicator, filterGroup, MOCK_RESOLUTION_MAP)).toEqual(false);
 
     filterGroup = {
       mode: 'and',
@@ -39,8 +43,8 @@ describe('Stix Filtering', () => {
       filterGroups: [],
     };
 
-    expect(await isStixMatchFilterGroup(testContext, ADMIN_USER, stixReport, filterGroup)).toEqual(true);
-    expect(await isStixMatchFilterGroup(testContext, ADMIN_USER, stixIndicator, filterGroup)).toEqual(true);
+    expect(await isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixReport, filterGroup, MOCK_RESOLUTION_MAP)).toEqual(true);
+    expect(await isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixIndicator, filterGroup, MOCK_RESOLUTION_MAP)).toEqual(true);
   });
 
   it('prevent access to stix object according to marking', async () => {
@@ -56,7 +60,7 @@ describe('Stix Filtering', () => {
     };
 
     const WHITE_USER = buildStandardUser([WHITE_TLP]);
-    expect(await isStixMatchFilterGroup(testContext, WHITE_USER, stixReport, filterGroup)).toEqual(false);
+    expect(await isStixMatchFilterGroup_MockableForUnitTests(testContext, WHITE_USER, stixReport, filterGroup, MOCK_RESOLUTION_MAP)).toEqual(false);
   });
 
   // TODO: add test with resolution of labels/indicators in filterGroup
@@ -91,15 +95,20 @@ describe('Stix Filtering', () => {
           }, {
             key: ['objectLabel'],
             mode: 'or',
-            operator: 'not_nil',
-            values: []
+            operator: 'eq',
+            values: ['id-for-label-indicator']
+          }, {
+            key: ['objectMarking'],
+            mode: 'or',
+            operator: 'eq',
+            values: ['id-for-marking-tlp:green']
           }],
           filterGroups: [],
         },
       ],
     };
-    expect(await isStixMatchFilterGroup(testContext, ADMIN_USER, stixReport, filterGroup)).toEqual(false);
-    expect(await isStixMatchFilterGroup(testContext, ADMIN_USER, stixIndicator, filterGroup)).toEqual(true);
+    expect(await isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixReport, filterGroup, MOCK_RESOLUTION_MAP)).toEqual(false);
+    expect(await isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixIndicator, filterGroup, MOCK_RESOLUTION_MAP)).toEqual(true);
   });
 
   it('throws error when filter group is invalid', async () => {
@@ -111,7 +120,7 @@ describe('Stix Filtering', () => {
       ],
       filterGroups: [],
     };
-    await expect(() => isStixMatchFilterGroup(testContext, ADMIN_USER, stixReport, multipleKeys)).rejects.toThrowError('Stix filtering can only be executed on a unique filter key');
+    await expect(() => isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixReport, multipleKeys, MOCK_RESOLUTION_MAP)).rejects.toThrowError('Stix filtering can only be executed on a unique filter key');
 
     const multipleKeysNested: FilterGroup = {
       mode: 'and',
@@ -126,7 +135,7 @@ describe('Stix Filtering', () => {
       }],
     };
 
-    await expect(() => isStixMatchFilterGroup(testContext, ADMIN_USER, stixReport, multipleKeysNested)).rejects.toThrowError('Stix filtering can only be executed on a unique filter key');
+    await expect(() => isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixReport, multipleKeysNested, MOCK_RESOLUTION_MAP)).rejects.toThrowError('Stix filtering can only be executed on a unique filter key');
 
     const unhandledKeys: FilterGroup = {
       mode: 'and',
@@ -137,7 +146,7 @@ describe('Stix Filtering', () => {
       filterGroups: [],
     };
 
-    await expect(() => isStixMatchFilterGroup(testContext, ADMIN_USER, stixReport, unhandledKeys)).rejects.toThrowError('Stix filtering is not compatible with the provided filter key');
+    await expect(() => isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixReport, unhandledKeys, MOCK_RESOLUTION_MAP)).rejects.toThrowError('Stix filtering is not compatible with the provided filter key');
 
     const unhandledKeysNested: FilterGroup = {
       mode: 'and',
@@ -152,6 +161,6 @@ describe('Stix Filtering', () => {
       }],
     };
 
-    await expect(() => isStixMatchFilterGroup(testContext, ADMIN_USER, stixReport, unhandledKeysNested)).rejects.toThrowError('Stix filtering is not compatible with the provided filter key');
+    await expect(() => isStixMatchFilterGroup_MockableForUnitTests(testContext, ADMIN_USER, stixReport, unhandledKeysNested, MOCK_RESOLUTION_MAP)).rejects.toThrowError('Stix filtering is not compatible with the provided filter key');
   });
 });