Skip to content

Commit

Permalink
[backend/frontend] fromOrTo relations filters with different operator…
Browse files Browse the repository at this point in the history 
…s/modes combinations (#6390)
  • Loading branch information
@Archidoit
Archidoit committed Mar 21, 2024
1 parent 4cd8e99 commit 0330878
Show file tree
Hide file tree
Showing 7 changed files with 456 additions and 40 deletions.
4 changes: 3 additions & 1 deletion ...ont/src/private/components/common/stix_domain_objects/StixDomainObjectThreatKnowledge.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,9 +445,10 @@ StixDomainObjectThreatKnowledgeProps
'createdBy',
'objectLabel',
'created',
'targets',
'toId',
]}
handleAddFilter={helpers.handleAddFilter}
searchContext={{ entityTypes: ['stix-core-relationship'] }}
/>
<IconButton color="primary" onClick={handleOpenTimeField} size="small">
<SettingsOutlined fontSize="small" />
Expand Down Expand Up @@ -508,6 +509,7 @@ StixDomainObjectThreatKnowledgeProps
handleRemoveFilter={helpers.handleRemoveFilter}
handleSwitchGlobalMode={helpers.handleSwitchGlobalMode}
handleSwitchLocalMode={helpers.handleSwitchLocalMode}
entityTypes={['stix-core-relationship']}
/>
<QueryRenderer
query={stixDomainObjectThreatKnowledgeStixRelationshipsQuery}
Expand Down
143 changes: 123 additions & 20 deletions opencti-platform/opencti-graphql/src/database/engine.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ import {
complexConversionFilterKeys,
COMPUTED_RELIABILITY_FILTER,
IDS_FILTER,
INSTANCE_FILTER_TARGET_TYPES,
INSTANCE_RELATION_TYPES_FILTER,
INSTANCE_REGARDING_OF,
INSTANCE_RELATION_FILTER,
RELATION_FROM_FILTER,
Expand Down Expand Up @@ -1611,22 +1611,49 @@ const buildLocalMustFilter = async (validFilter) => {
const nestedShould = [];
const nestedFieldKey = `${parentKey}.${nestedKey}`;
if (nestedKey === ID_INTERNAL) {
if (nestedOperator === 'not_eq') {
if (nestedOperator === 'nil') {
nestedMustNot.push({
exists: {
field: nestedFieldKey
}
});
} else if (nestedOperator === 'not_nil') {
nestedShould.push({
exists: {
field: nestedFieldKey
}
});
} else if (nestedOperator === 'not_eq') {
nestedMustNot.push({ terms: { [`${nestedFieldKey}.keyword`]: nestedValues } });
} else {
} else { // nestedOperator = 'eq'
nestedShould.push({ terms: { [`${nestedFieldKey}.keyword`]: nestedValues } });
}
} else {
for (let i = 0; i < nestedValues.length; i += 1) {
const nestedSearchValues = nestedValues[i].toString();
if (nestedOperator === 'wildcard') {
nestedShould.push({ query_string: { query: `${nestedSearchValues}`, fields: [nestedFieldKey] } });
} else if (nestedOperator === 'not_eq') {
nestedMustNot.push({ match_phrase: { [nestedFieldKey]: nestedSearchValues } });
} else if (RANGE_OPERATORS.includes(nestedOperator)) {
nestedShould.push({ range: { [nestedFieldKey]: { [nestedOperator]: nestedSearchValues } } });
} else {
nestedShould.push({ match_phrase: { [nestedFieldKey]: nestedSearchValues } });
} else { // nested key !== internal_id
// eslint-disable-next-line no-lonely-if
if (nestedOperator === 'nil') {
nestedMustNot.push({
exists: {
field: nestedFieldKey
}
});
} else if (nestedOperator === 'not_nil') {
nestedShould.push({
exists: {
field: nestedFieldKey
}
});
} else {
for (let i = 0; i < nestedValues.length; i += 1) {
const nestedSearchValues = nestedValues[i].toString();
if (nestedOperator === 'wildcard') {
nestedShould.push({ query_string: { query: `${nestedSearchValues}`, fields: [nestedFieldKey] } });
} else if (nestedOperator === 'not_eq') {
nestedMustNot.push({ match_phrase: { [nestedFieldKey]: nestedSearchValues } });
} else if (RANGE_OPERATORS.includes(nestedOperator)) {
nestedShould.push({ range: { [nestedFieldKey]: { [nestedOperator]: nestedSearchValues } } });
} else { // nestedOperator = 'eq'
nestedShould.push({ match_phrase: { [nestedFieldKey]: nestedSearchValues } });
}
}
}
}
Expand Down Expand Up @@ -2023,6 +2050,78 @@ const adaptFilterToSourceReliabilityFilterKey = async (context, user, filter) =>
return { newFilter, newFilterGroup };
};

// fromOrToId and elementWithTargetTypes filters
// are composed of a condition on fromId/fromType and a condition on toId/toType of a relationship
const adaptFilterToFromOrToFilterKeys = (filter) => {
const { key, operator = 'eq', mode = 'or', values } = filter;
const arrayKeys = Array.isArray(key) ? key : [key];
if (arrayKeys.length > 1) {
throw UnsupportedError('A filter with these multiple keys is not supported', { keys: arrayKeys });
}
let nestedKey;
if (arrayKeys[0] === INSTANCE_RELATION_TYPES_FILTER) {
nestedKey = 'types';
} else if (arrayKeys[0] === INSTANCE_RELATION_FILTER) {
nestedKey = 'internal_id';
} else {
throw UnsupportedError('A related relations filter with this key is not supported', { key: arrayKeys[0] });
}

let newFilterGroup;
// define mode for the filter group
let globalMode = 'or';
if (operator === 'eq' || operator === 'not_nil') {
// relatedType = malware <-> fromType = malware OR toType = malware
// relatedType is not empty <-> fromType is not empty OR toType is not empty
globalMode = 'or';
} else if (operator === 'not_eq' || operator === 'nil') {
// relatedType != malware <-> fromType != malware AND toType != malware
// relatedType is empty <-> fromType is empty AND toType is empty
globalMode = 'and';
} else {
throw Error(`${INSTANCE_RELATION_TYPES_FILTER} filter only support 'eq', 'not_eq', 'nil' and 'not_nil' operators, not ${operator}.`);
}
// define the filter group
if (operator === 'eq' || operator === 'not_eq') {
const filterGroupsForValues = values.map((val) => {
const nestedFrom = [
{ key: nestedKey, operator, values: [val] },
{ key: 'role', operator: 'wildcard', values: ['*_from'] }
];
const nestedTo = [
{ key: nestedKey, operator, values: [val] },
{ key: 'role', operator: 'wildcard', values: ['*_to'] }
];
return {
mode: globalMode,
filters: [{ key: 'connections', nested: nestedFrom, mode }, { key: 'connections', nested: nestedTo, mode }],
filterGroups: [],
};
});
newFilterGroup = {
mode,
filters: [],
filterGroups: filterGroupsForValues,
};
} else if (operator === 'nil' || operator === 'not_nil') {
const nestedFrom = [
{ key: nestedKey, operator, values: [] },
{ key: 'role', operator: 'wildcard', values: ['*_from'] }
];
const nestedTo = [
{ key: nestedKey, operator, values: [] },
{ key: 'role', operator: 'wildcard', values: ['*_to'] }
];
const innerFilters = [{ key: 'connections', nested: nestedFrom, mode }, { key: 'connections', nested: nestedTo, mode }];
newFilterGroup = {
mode: globalMode,
filters: innerFilters,
filterGroups: [],
};
}
return { newFilter: undefined, newFilterGroup };
};

const adaptFilterToComputedReliabilityFilterKey = async (context, user, filter) => {
const { key, operator = 'eq' } = filter;
const arrayKeys = Array.isArray(key) ? key : [key];
Expand Down Expand Up @@ -2270,8 +2369,10 @@ const completeSpecialFilterKeys = async (context, user, inputFilters) => {
}
}
if (filterKey === INSTANCE_RELATION_FILTER) {
const nested = [{ key: 'internal_id', operator: filter.operator, values: filter.values }];
finalFilters.push({ key: 'connections', nested, mode: filter.mode });
const { newFilterGroup } = adaptFilterToFromOrToFilterKeys(filter);
if (newFilterGroup) {
finalFilterGroups.push(newFilterGroup);
}
}
if (filterKey === RELATION_FROM_FILTER || filterKey === RELATION_TO_FILTER || filterKey === RELATION_TO_SIGHTING_FILTER) {
const side = filterKey === RELATION_FROM_FILTER ? 'from' : 'to';
Expand All @@ -2289,9 +2390,11 @@ const completeSpecialFilterKeys = async (context, user, inputFilters) => {
];
finalFilters.push({ key: 'connections', nested, mode: filter.mode });
}
if (filterKey === INSTANCE_FILTER_TARGET_TYPES) {
const nested = [{ key: 'types', operator: filter.operator, values: filter.values }];
finalFilters.push({ key: 'connections', nested, mode: filter.mode });
if (filterKey === INSTANCE_RELATION_TYPES_FILTER) {
const { newFilterGroup } = adaptFilterToFromOrToFilterKeys(filter);
if (newFilterGroup) {
finalFilterGroups.push(newFilterGroup);
}
}
if (filterKey === RELATION_FROM_ROLE_FILTER || filterKey === RELATION_TO_ROLE_FILTER) {
const side = filterKey === RELATION_FROM_ROLE_FILTER ? 'from' : 'to';
Expand All @@ -2303,7 +2406,7 @@ const completeSpecialFilterKeys = async (context, user, inputFilters) => {
if (filterKey === ALIAS_FILTER) {
finalFilters.push({ ...filter, key: [ATTRIBUTE_ALIASES, ATTRIBUTE_ALIASES_OPENCTI] });
}
} else if (arrayKeys.some((fiterKey) => isObjectAttribute(fiterKey)) && !arrayKeys.some((fiterKey) => fiterKey === 'connections')) {
} else if (arrayKeys.some((filterKey) => isObjectAttribute(filterKey)) && !arrayKeys.some((filterKey) => filterKey === 'connections')) {
if (arrayKeys.length > 1) {
throw UnsupportedError('A filter with these multiple keys is not supported', { keys: arrayKeys });
}
Expand Down
15 changes: 11 additions & 4 deletions ...atform/opencti-graphql/src/modules/attributes/basicRelationship-registrationAttributes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,14 @@ import {
} from '../../schema/attribute-definition';
import { schemaAttributesDefinition } from '../../schema/schema-attributes';
import { ABSTRACT_BASIC_RELATIONSHIP } from '../../schema/general';
import { INSTANCE_RELATION_FILTER } from '../../utils/filtering/filtering-constants';
import {
INSTANCE_RELATION_TYPES_FILTER,
INSTANCE_RELATION_FILTER,
RELATION_FROM_FILTER,
RELATION_FROM_TYPES_FILTER,
RELATION_TO_FILTER,
RELATION_TO_TYPES_FILTER
} from '../../utils/filtering/filtering-constants';

export const connections: AttributeDefinition = {
name: 'connections',
Expand All @@ -30,14 +37,14 @@ export const connections: AttributeDefinition = {
mappings: [
{ ...internalId as IdAttribute,
associatedFilterKeys: [
{ key: 'fromId', label: 'Source entity' },
{ key: 'toId', label: 'Target entity' },
{ key: RELATION_FROM_FILTER, label: 'Source entity' },
{ key: RELATION_TO_FILTER, label: 'Target entity' },
{ key: INSTANCE_RELATION_FILTER, label: 'Related entity' }
]
},
{ name: 'name', label: 'Name', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false },
{ name: 'role', label: 'Role', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false },
{ name: 'types', label: 'Types', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false, associatedFilterKeys: [{ key: 'fromTypes', label: 'Source type' }, { key: 'toTypes', label: 'Target type' }] },
{ name: 'types', label: 'Types', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false, associatedFilterKeys: [{ key: RELATION_FROM_TYPES_FILTER, label: 'Source type' }, { key: RELATION_TO_TYPES_FILTER, label: 'Target type' }, { key: INSTANCE_RELATION_TYPES_FILTER, label: 'Related type' }] },
],
};

Expand Down
15 changes: 11 additions & 4 deletions ...orm/opencti-graphql/src/modules/attributes/stixCoreRelationship-registrationAttributes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,14 @@ import { schemaAttributesDefinition } from '../../schema/schema-attributes';
import { STIX_CORE_RELATIONSHIPS } from '../../schema/stixCoreRelationship';
import { connections } from './basicRelationship-registrationAttributes';
import { internalId } from '../../schema/attribute-definition';
import { INSTANCE_RELATION_FILTER } from '../../utils/filtering/filtering-constants';
import {
INSTANCE_RELATION_TYPES_FILTER,
INSTANCE_RELATION_FILTER,
RELATION_FROM_FILTER,
RELATION_FROM_TYPES_FILTER,
RELATION_TO_FILTER,
RELATION_TO_TYPES_FILTER
} from '../../utils/filtering/filtering-constants';

export const stixCoreRelationshipsAttributes: Array<AttributeDefinition> = [
{ ...connections as NestedObjectAttribute,
Expand All @@ -14,14 +21,14 @@ export const stixCoreRelationshipsAttributes: Array<AttributeDefinition> = [
isFilterable: true,
entityTypes: [ABSTRACT_STIX_CORE_OBJECT],
associatedFilterKeys: [
{ key: 'fromId', label: 'Source entity' },
{ key: 'toId', label: 'Target entity' },
{ key: RELATION_FROM_FILTER, label: 'Source entity' },
{ key: RELATION_TO_FILTER, label: 'Target entity' },
{ key: INSTANCE_RELATION_FILTER, label: 'Related entity' }
]
},
{ name: 'name', label: 'Name', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false },
{ name: 'role', label: 'Role', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false },
{ name: 'types', label: 'Types', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: true, associatedFilterKeys: [{ key: 'fromTypes', label: 'Source type' }, { key: 'toTypes', label: 'Target type' }] },
{ name: 'types', label: 'Types', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: true, associatedFilterKeys: [{ key: RELATION_FROM_TYPES_FILTER, label: 'Source type' }, { key: RELATION_TO_TYPES_FILTER, label: 'Target type' }, { key: INSTANCE_RELATION_TYPES_FILTER, label: 'Related type' }] },
],
},
{ name: 'entity_type', label: 'Entity type', type: 'string', format: 'short', editDefault: false, mandatoryType: 'no', multiple: true, upsert: true, isFilterable: false },
Expand Down
4 changes: 2 additions & 2 deletions opencti-platform/opencti-graphql/src/utils/filtering/filtering-constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export const RELATION_FROM_ROLE_FILTER = 'fromRole';
export const RELATION_TO_ROLE_FILTER = 'toRole';
export const RELATION_FROM_TYPES_FILTER = 'fromTypes';
export const RELATION_TO_TYPES_FILTER = 'toTypes';
export const INSTANCE_FILTER_TARGET_TYPES = 'elementWithTargetTypes'; // TODO Rename/migrate to fromOrToType
export const INSTANCE_RELATION_TYPES_FILTER = 'elementWithTargetTypes'; // TODO Rename/migrate to fromOrToType
export const CONNECTED_TO_INSTANCE_FILTER = 'connectedToId'; // TODO Rename/migrate to triggerListenId
export const CONNECTED_TO_INSTANCE_SIDE_EVENTS_FILTER = 'connectedToId_sideEvents';

Expand Down Expand Up @@ -75,7 +75,7 @@ export const complexConversionFilterKeys = [
SOURCE_RELIABILITY_FILTER, // reliability of the author
COMPUTED_RELIABILITY_FILTER, // reliability, or reliabilityof the author if no reliability
INSTANCE_RELATION_FILTER, // nested relation for the from or to of a relationship
INSTANCE_FILTER_TARGET_TYPES, // nested relation for the from or type type of a relationship
INSTANCE_RELATION_TYPES_FILTER, // nested relation for the from or to type of a relationship
RELATION_FROM_FILTER, // nested relation for the from of a relationship
RELATION_TO_FILTER, // nested relation for the to of a relationship
RELATION_TO_SIGHTING_FILTER, // nested sigthing relation for the to of a sighting
Expand Down
29 changes: 21 additions & 8 deletions opencti-platform/opencti-graphql/tests/01-unit/domain/filterKeysSchema-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,16 @@ import { ENTITY_TYPE_NOTIFICATION, ENTITY_TYPE_TRIGGER } from '../../../src/modu
import { ENTITY_HASHED_OBSERVABLE_ARTIFACT, ENTITY_HASHED_OBSERVABLE_STIX_FILE, ENTITY_HASHED_OBSERVABLE_X509_CERTIFICATE } from '../../../src/schema/stixCyberObservable';
import { ENTITY_TYPE_CONTAINER_CASE } from '../../../src/modules/case/case-types';
import { ENTITY_TYPE_CONTAINER_GROUPING } from '../../../src/modules/grouping/grouping-types';
import { ALIAS_FILTER, CONNECTED_TO_INSTANCE_FILTER, CONTEXT_OBJECT_LABEL_FILTER, INSTANCE_REGARDING_OF, TYPE_FILTER } from '../../../src/utils/filtering/filtering-constants';
import {
ALIAS_FILTER,
CONNECTED_TO_INSTANCE_FILTER,
CONTEXT_OBJECT_LABEL_FILTER,
INSTANCE_RELATION_TYPES_FILTER,
INSTANCE_REGARDING_OF,
RELATION_FROM_FILTER,
RELATION_TO_TYPES_FILTER,
TYPE_FILTER
} from '../../../src/utils/filtering/filtering-constants';
import { ENTITY_TYPE_HISTORY } from '../../../src/schema/internalObject';

describe('Filter keys schema generation testing', async () => {
Expand Down Expand Up @@ -128,22 +137,26 @@ describe('Filter keys schema generation testing', async () => {
expect(filterDefinition?.type).toEqual('enum');
expect(filterDefinition?.elementsForFilterValuesSearch.length).toEqual(3); // create, update, delete
});
it('should construct correct filter definition for nested object attributes', () => {
it('should construct correct filter definition for nested object attributes: case of relationships', () => {
// 'fromId' for stix core relationships
let filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CORE_RELATIONSHIP)?.get('fromId');
expect(filterDefinition?.filterKey).toEqual('fromId');
let filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CORE_RELATIONSHIP)?.get(RELATION_FROM_FILTER);
expect(filterDefinition?.filterKey).toEqual(RELATION_FROM_FILTER);
expect(filterDefinition?.type).toEqual('id');
expect(filterDefinition?.label).toEqual('Source entity');
expect(filterDefinition?.elementsForFilterValuesSearch.length).toEqual(1);
expect(filterDefinition?.elementsForFilterValuesSearch[0]).toEqual(ABSTRACT_STIX_CORE_OBJECT);
// 'toTypes' for stix core relationships
filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CORE_RELATIONSHIP)?.get('toTypes');
expect(filterDefinition?.filterKey).toEqual('toTypes');
filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CORE_RELATIONSHIP)?.get(RELATION_TO_TYPES_FILTER);
expect(filterDefinition?.filterKey).toEqual(RELATION_TO_TYPES_FILTER);
expect(filterDefinition?.type).toEqual('string');
expect(filterDefinition?.label).toEqual('Target type');
expect(filterDefinition?.elementsForFilterValuesSearch.length).toEqual(0);
// 'elementWithTargetTypes' for stix core relationships
filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CORE_RELATIONSHIP)?.get(INSTANCE_RELATION_TYPES_FILTER);
expect(filterDefinition?.filterKey).toEqual(INSTANCE_RELATION_TYPES_FILTER);
expect(filterDefinition?.type).toEqual('string');
// 'fromId' for basic relationships: not filterable
filterDefinition = filterKeysSchema.get(ABSTRACT_BASIC_RELATIONSHIP)?.get('fromId');
filterDefinition = filterKeysSchema.get(ABSTRACT_BASIC_RELATIONSHIP)?.get(RELATION_FROM_FILTER);
expect(filterDefinition).toBeUndefined();
});
it('should construct correct filter definition for special filter keys', () => {
Expand Down Expand Up @@ -199,7 +212,7 @@ describe('Filter keys schema generation testing', async () => {
expect(filterDefinition?.subEntityTypes.length).toEqual(1); // 'Threat-Actor-Individual'

// Stix Core Relationships
filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CORE_RELATIONSHIP)?.get('fromId');
filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CORE_RELATIONSHIP)?.get(RELATION_FROM_FILTER);
expect(filterDefinition?.subEntityTypes.length).toEqual(49); // 48 stix core relationship types + abstract type 'stix-core-relationships'
// Stix Cyber Observables
filterDefinition = filterKeysSchema.get(ABSTRACT_STIX_CYBER_OBSERVABLE)?.get('x_opencti_score'); // attribute existing for all the observables
Expand Down

0 comments on commit 0330878

Please sign in to comment.