Skip to content

Commit

Permalink
[backend] Improve handling of refs relationships (#4476)
Browse files Browse the repository at this point in the history
  • Loading branch information
richard-julien committed Nov 30, 2023
1 parent 7358b11 commit 7c59f84
Show file tree
Hide file tree
Showing 8 changed files with 135 additions and 64 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -5794,6 +5794,8 @@ input NetworkTrafficAddInput {
start: DateTime
end: DateTime
is_active: Boolean
src: String
dst: String
src_port: Int
dst_port: Int
protocols: [String]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8356,6 +8356,8 @@ input NetworkTrafficAddInput {
start: DateTime
end: DateTime
is_active: Boolean
src: String
dst: String
src_port: Int
dst_port: Int
protocols: [String]
Expand Down
23 changes: 15 additions & 8 deletions opencti-platform/opencti-graphql/src/database/middleware.js
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,9 @@ const loadElementMetaDependencies = async (context, user, elements, args = {}) =
return resolvedElement ? { ...resolvedElement, i_relation: v } : {};
}, values).filter((d) => isNotEmptyField(d));
const metaRefKey = schemaRelationsRefDefinition.getRelationRef(element.entity_type, inputKey);
if (isEmptyField(metaRefKey)) {
throw UnsupportedError('SCHEMA_VALIDATION_ERROR', { key, inputKey, type: element.entity_type });
}
const invalidRelations = values.filter((v) => toResolvedElements[v.toId] === undefined);
if (invalidRelations.length > 0) {
// Some targets can be unresolved in case of potential inconsistency between relation and target
Expand Down Expand Up @@ -2005,12 +2008,10 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs
const previous = currentValue ? [currentValue] : currentValue;
updatedInputs.push({ key, value: [targetCreated], previous });
updatedInstance[key] = targetCreated;
updatedInstance[relType] = updatedInstance[key].internal_id;
} else if (currentValue) {
// Just replace by nothing
updatedInputs.push({ key, value: null, previous: [currentValue] });
updatedInstance[key] = null;
updatedInstance[relType] = null;
}
}
} else {
Expand Down Expand Up @@ -2039,7 +2040,6 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs
}
updatedInputs.push({ key, value: refs, previous: updatedInstance[key] });
updatedInstance[key] = refs;
updatedInstance[relType] = updatedInstance[key].map((t) => t.internal_id);
}
}
if (operation === UPDATE_OPERATION_ADD) {
Expand All @@ -2050,7 +2050,6 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs
relationsToCreate.push(...newRelations);
updatedInputs.push({ key, value: refsToCreate, operation });
updatedInstance[key] = [...(updatedInstance[key] || []), ...refsToCreate];
updatedInstance[relType] = updatedInstance[key].map((t) => t.internal_id);
}
}
if (operation === UPDATE_OPERATION_REMOVE) {
Expand All @@ -2061,7 +2060,6 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs
relationsToDelete.push(...relsToDelete);
updatedInputs.push({ key, value: refs, operation });
updatedInstance[key] = (updatedInstance[key] || []).filter((c) => !targetIds.includes(c.internal_id));
updatedInstance[relType] = updatedInstance[key].map((t) => t.internal_id);
}
}
}
Expand Down Expand Up @@ -2117,10 +2115,10 @@ export const updateAttributeMetaResolved = async (context, user, initial, inputs
if (lock) await lock.unlock();
}
};
export const updateAttribute = async (context, user, id, type, inputs, opts = {}) => {
const initial = await storeLoadByIdWithRefs(context, user, id, { type });

export const updateAttributeFromLoadedWithRefs = async (context, user, initial, inputs, opts = {}) => {
if (!initial) {
throw FunctionalError('Cant find element to update', { id, type });
throw FunctionalError('Cant update undefined element');
}
const updates = Array.isArray(inputs) ? inputs : [inputs];
const metaKeys = [...schemaRelationsRefDefinition.getStixNames(initial.entity_type), ...schemaRelationsRefDefinition.getInputNames(initial.entity_type)];
Expand All @@ -2136,6 +2134,15 @@ export const updateAttribute = async (context, user, id, type, inputs, opts = {}
});
return updateAttributeMetaResolved(context, user, initial, revolvedInputs, opts);
};

export const updateAttribute = async (context, user, id, type, inputs, opts = {}) => {
const initial = await storeLoadByIdWithRefs(context, user, id, { type });
if (!initial) {
throw FunctionalError('Cant find element to update', { id, type });
}
return updateAttributeFromLoadedWithRefs(context, user, initial, inputs, opts);
};

export const patchAttribute = async (context, user, id, type, patch, opts = {}) => {
const inputs = transformPatchToInput(patch, opts.operations);
return updateAttribute(context, user, id, type, inputs, opts);
Expand Down
30 changes: 16 additions & 14 deletions opencti-platform/opencti-graphql/src/domain/file.js
Original file line number Diff line number Diff line change
Expand Up @@ -103,20 +103,22 @@ export const askJobImport = async (context, user, args) => {
entity_name: entityName,
entity_type: entityType
};
if (entity.creator_id) {
contextData.creator_ids = Array.isArray(entity.creator_id) ? entity.creator_id : [entity.creator_id];
}
if (entity[RELATION_GRANTED_TO]) {
contextData.granted_refs_ids = entity[RELATION_GRANTED_TO];
}
if (entity[RELATION_OBJECT_MARKING]) {
contextData.object_marking_refs_ids = entity[RELATION_OBJECT_MARKING];
}
if (entity[RELATION_CREATED_BY]) {
contextData.created_by_ref_id = entity[RELATION_CREATED_BY];
}
if (entity[RELATION_OBJECT_LABEL]) {
contextData.labels_ids = entity[RELATION_OBJECT_LABEL];
if (entity) { // Entity can be null for global
if (entity.creator_id) {
contextData.creator_ids = Array.isArray(entity.creator_id) ? entity.creator_id : [entity.creator_id];
}
if (entity[RELATION_GRANTED_TO]) {
contextData.granted_refs_ids = entity[RELATION_GRANTED_TO];
}
if (entity[RELATION_OBJECT_MARKING]) {
contextData.object_marking_refs_ids = entity[RELATION_OBJECT_MARKING];
}
if (entity[RELATION_CREATED_BY]) {
contextData.created_by_ref_id = entity[RELATION_CREATED_BY];
}
if (entity[RELATION_OBJECT_LABEL]) {
contextData.labels_ids = entity[RELATION_OBJECT_LABEL];
}
}
await publishUserAction({
user,
Expand Down
45 changes: 38 additions & 7 deletions opencti-platform/opencti-graphql/src/domain/stixRefRelationship.js
Original file line number Diff line number Diff line change
@@ -1,17 +1,26 @@
import { dissoc, propOr } from 'ramda';
import { createRelation, deleteElementById, updateAttribute } from '../database/middleware';
import {
deleteElementById,
storeLoadByIdWithRefs,
updateAttribute,
updateAttributeFromLoadedWithRefs
} from '../database/middleware';
import { BUS_TOPICS } from '../config/conf';
import { notify } from '../database/redis';
import { ABSTRACT_STIX_REF_RELATIONSHIP, ABSTRACT_STIX_RELATIONSHIP } from '../schema/general';
import { FunctionalError } from '../config/errors';
import { isStixRefRelationship, META_RELATIONS, STIX_REF_RELATIONSHIP_TYPES } from '../schema/stixRefRelationship';
import { listRelations, storeLoadById } from '../database/middleware-loader';
import { internalLoadById, listRelations, storeLoadById } from '../database/middleware-loader';
import { stixCoreRelationshipCleanContext, stixCoreRelationshipEditContext } from './stixCoreRelationship';
import { schemaTypesDefinition } from '../schema/schema-types';
import { schemaRelationsRefDefinition } from '../schema/schema-relationsRef';
import { findById as findStixObjectOrStixRelationshipById } from './stixObjectOrStixRelationship';
import { elCount } from '../database/engine';
import { READ_INDEX_STIX_CYBER_OBSERVABLE_RELATIONSHIPS, READ_INDEX_STIX_META_RELATIONSHIPS } from '../database/utils';
import {
READ_INDEX_STIX_CYBER_OBSERVABLE_RELATIONSHIPS,
READ_INDEX_STIX_META_RELATIONSHIPS,
UPDATE_OPERATION_ADD
} from '../database/utils';

// Query

Expand Down Expand Up @@ -48,14 +57,36 @@ export const isDatable = (entityType, relationshipType) => {
};

// Mutation

// @Deprecated method.
// updateField must be directly used
export const addStixRefRelationship = async (context, user, stixRefRelationship) => {
if (!isStixRefRelationship(stixRefRelationship.relationship_type)) {
throw FunctionalError(`Only ${ABSTRACT_STIX_REF_RELATIONSHIP} can be added through this method, got ${stixRefRelationship.relationship_type}.`);
}
return createRelation(context, user, stixRefRelationship).then((relationData) => {
return notify(BUS_TOPICS[ABSTRACT_STIX_REF_RELATIONSHIP].ADDED_TOPIC, relationData, user);
});
const fromPromise = storeLoadByIdWithRefs(context, user, stixRefRelationship.fromId);
const toPromise = internalLoadById(context, user, stixRefRelationship.toId);
const [from, to] = await Promise.all([fromPromise, toPromise]);
if (!from || !to) {
throw FunctionalError('MISSING_ELEMENTS', {
from: stixRefRelationship.fromId,
from_missing: !from,
to: stixRefRelationship.toId,
to_missing: !to
});
}
const refInputName = schemaRelationsRefDefinition.convertDatabaseNameToInputName(from.entity_type, stixRefRelationship.relationship_type);
const inputs = [{ key: refInputName, value: [stixRefRelationship.toId], operation: UPDATE_OPERATION_ADD }];
await updateAttributeFromLoadedWithRefs(context, user, from, inputs);
const opts = {
first: 1,
connectionFormat: false,
fromId: from.internal_id,
toId: to.internal_id,
orderBy: 'created_at',
orderMode: 'desc'
};
const lastCreatedRef = await listRelations(context, user, stixRefRelationship.relationship_type, opts);
return notify(BUS_TOPICS[ABSTRACT_STIX_REF_RELATIONSHIP].ADDED_TOPIC, lastCreatedRef[0], user);
};
export const stixRefRelationshipEditField = async (context, user, stixRefRelationshipId, input) => {
// Not use ABSTRACT_STIX_REF_RELATIONSHIP to have compatibility on parent type with ABSTRACT_STIX_CYBER_OBSERVABLE_RELATIONSHIP type
Expand Down
2 changes: 2 additions & 0 deletions opencti-platform/opencti-graphql/src/generated/graphql.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14504,13 +14504,15 @@ export type NetworkTrafficStixCoreRelationshipsDistributionArgs = {
};

export type NetworkTrafficAddInput = {
dst?: InputMaybe<Scalars['String']['input']>;
dst_byte_count?: InputMaybe<Scalars['Int']['input']>;
dst_packets?: InputMaybe<Scalars['Int']['input']>;
dst_port?: InputMaybe<Scalars['Int']['input']>;
end?: InputMaybe<Scalars['DateTime']['input']>;
file?: InputMaybe<Scalars['Upload']['input']>;
is_active?: InputMaybe<Scalars['Boolean']['input']>;
protocols?: InputMaybe<Array<InputMaybe<Scalars['String']['input']>>>;
src?: InputMaybe<Scalars['String']['input']>;
src_byte_count?: InputMaybe<Scalars['Int']['input']>;
src_packets?: InputMaybe<Scalars['Int']['input']>;
src_port?: InputMaybe<Scalars['Int']['input']>;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,56 +63,56 @@ schemaRelationsRefDefinition.registerRelationsRef(
);

schemaRelationsRefDefinition.registerRelationsRef(ENTITY_DIRECTORY, [
buildRelationRef(contains, (fromType: string, toType: string) => [ENTITY_DIRECTORY, ENTITY_HASHED_OBSERVABLE_STIX_FILE].includes(toType))
buildRelationRef(contains, (_: string, toType: string) => [ENTITY_DIRECTORY, ENTITY_HASHED_OBSERVABLE_STIX_FILE].includes(toType))
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_DOMAIN_NAME, [
buildRelationRef(resolvesTo, (fromType: string, toType: string) => [ENTITY_DOMAIN_NAME, ENTITY_IPV4_ADDR, ENTITY_IPV6_ADDR].includes(toType))
buildRelationRef(resolvesTo, (_: string, toType: string) => [ENTITY_DOMAIN_NAME, ENTITY_IPV4_ADDR, ENTITY_IPV6_ADDR].includes(toType))
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_EMAIL_MESSAGE, [
buildRelationRef(from, (fromType: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(sender, (fromType: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(to, (fromType: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(cc, (fromType: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(bcc, (fromType: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(bodyMultipart, (fromType: string, toType: string) => ENTITY_EMAIL_MIME_PART_TYPE === toType),
buildRelationRef(rawEmail, (fromType: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
buildRelationRef(from, (_: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(sender, (_: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(to, (_: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(cc, (_: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(bcc, (_: string, toType: string) => ENTITY_EMAIL_ADDR === toType),
buildRelationRef(bodyMultipart, (_: string, toType: string) => ENTITY_EMAIL_MIME_PART_TYPE === toType),
buildRelationRef(rawEmail, (_: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_EMAIL_ADDR, [
buildRelationRef(belongsTo, (fromType: string, toType: string) => ENTITY_USER_ACCOUNT === toType)
buildRelationRef(belongsTo, (_: string, toType: string) => ENTITY_USER_ACCOUNT === toType)
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_EMAIL_MIME_PART_TYPE, [
buildRelationRef(bodyRaw, (fromType: string, toType: string) => [ENTITY_HASHED_OBSERVABLE_ARTIFACT, ENTITY_HASHED_OBSERVABLE_STIX_FILE].includes(toType))
buildRelationRef(bodyRaw, (_: string, toType: string) => [ENTITY_HASHED_OBSERVABLE_ARTIFACT, ENTITY_HASHED_OBSERVABLE_STIX_FILE].includes(toType))
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_HASHED_OBSERVABLE_STIX_FILE, [
buildRelationRef(contains, (fromType: string, toType: string) => getParentTypes(toType).includes(ABSTRACT_STIX_CYBER_OBSERVABLE)),
buildRelationRef(parentDirectory, (fromType: string, toType: string) => ENTITY_DIRECTORY === toType),
buildRelationRef(obsContent, (fromType: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
buildRelationRef(contains, (_: string, toType: string) => getParentTypes(toType).includes(ABSTRACT_STIX_CYBER_OBSERVABLE)),
buildRelationRef(parentDirectory, (_: string, toType: string) => ENTITY_DIRECTORY === toType),
buildRelationRef(obsContent, (_: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_IPV4_ADDR, [
buildRelationRef(resolvesTo, (fromType: string, toType: string) => ENTITY_MAC_ADDR === toType),
buildRelationRef(belongsTo, (fromType: string, toType: string) => ENTITY_AUTONOMOUS_SYSTEM === toType),
buildRelationRef(resolvesTo, (_: string, toType: string) => ENTITY_MAC_ADDR === toType),
buildRelationRef(belongsTo, (_: string, toType: string) => ENTITY_AUTONOMOUS_SYSTEM === toType),
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_IPV6_ADDR, [
buildRelationRef(resolvesTo, (fromType: string, toType: string) => ENTITY_MAC_ADDR === toType),
buildRelationRef(belongsTo, (fromType: string, toType: string) => ENTITY_AUTONOMOUS_SYSTEM === toType),
buildRelationRef(resolvesTo, (_: string, toType: string) => ENTITY_MAC_ADDR === toType),
buildRelationRef(belongsTo, (_: string, toType: string) => ENTITY_AUTONOMOUS_SYSTEM === toType),
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_NETWORK_TRAFFIC, [
buildRelationRef(src, (fromType: string, toType: string) => [ENTITY_DOMAIN_NAME, ENTITY_IPV4_ADDR, ENTITY_IPV6_ADDR, ENTITY_MAC_ADDR].includes(toType)),
buildRelationRef(dst, (fromType: string, toType: string) => [ENTITY_DOMAIN_NAME, ENTITY_IPV4_ADDR, ENTITY_IPV6_ADDR, ENTITY_MAC_ADDR].includes(toType)),
buildRelationRef(srcPayload, (fromType: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
buildRelationRef(dstPayload, (fromType: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
buildRelationRef(encapsulates, (fromType: string, toType: string) => ENTITY_NETWORK_TRAFFIC === toType),
buildRelationRef(encapsulatedBy, (fromType: string, toType: string) => ENTITY_NETWORK_TRAFFIC === toType),
buildRelationRef(src, (_: string, toType: string) => [ENTITY_DOMAIN_NAME, ENTITY_IPV4_ADDR, ENTITY_IPV6_ADDR, ENTITY_MAC_ADDR].includes(toType)),
buildRelationRef(dst, (_: string, toType: string) => [ENTITY_DOMAIN_NAME, ENTITY_IPV4_ADDR, ENTITY_IPV6_ADDR, ENTITY_MAC_ADDR].includes(toType)),
buildRelationRef(srcPayload, (_: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
buildRelationRef(dstPayload, (_: string, toType: string) => ENTITY_HASHED_OBSERVABLE_ARTIFACT === toType),
buildRelationRef(encapsulates, (_: string, toType: string) => ENTITY_NETWORK_TRAFFIC === toType),
buildRelationRef(encapsulatedBy, (_: string, toType: string) => ENTITY_NETWORK_TRAFFIC === toType),
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_PROCESS, [
buildRelationRef(openedConnections, (fromType: string, toType: string) => ENTITY_NETWORK_TRAFFIC === toType),
buildRelationRef(creatorUser, (fromType: string, toType: string) => ENTITY_USER_ACCOUNT === toType),
buildRelationRef(image, (fromType: string, toType: string) => ENTITY_HASHED_OBSERVABLE_STIX_FILE === toType),
buildRelationRef(parent, (fromType: string, toType: string) => ENTITY_PROCESS === toType),
buildRelationRef(child, (fromType: string, toType: string) => ENTITY_PROCESS === toType),
buildRelationRef(serviceDlls, (fromType: string, toType: string) => ENTITY_HASHED_OBSERVABLE_STIX_FILE === toType),
buildRelationRef(openedConnections, (_: string, toType: string) => ENTITY_NETWORK_TRAFFIC === toType),
buildRelationRef(creatorUser, (_: string, toType: string) => ENTITY_USER_ACCOUNT === toType),
buildRelationRef(image, (_: string, toType: string) => ENTITY_HASHED_OBSERVABLE_STIX_FILE === toType),
buildRelationRef(parent, (_: string, toType: string) => ENTITY_PROCESS === toType),
buildRelationRef(child, (_: string, toType: string) => ENTITY_PROCESS === toType),
buildRelationRef(serviceDlls, (_: string, toType: string) => ENTITY_HASHED_OBSERVABLE_STIX_FILE === toType),
]);
schemaRelationsRefDefinition.registerRelationsRef(ENTITY_WINDOWS_REGISTRY_KEY, [
buildRelationRef(values, (fromType: string, toType: string) => ENTITY_WINDOWS_REGISTRY_VALUE_TYPE === toType),
buildRelationRef(creatorUser, (fromType: string, toType: string) => ENTITY_USER_ACCOUNT === toType),
buildRelationRef(values, (_: string, toType: string) => ENTITY_WINDOWS_REGISTRY_VALUE_TYPE === toType),
buildRelationRef(creatorUser, (_: string, toType: string) => ENTITY_USER_ACCOUNT === toType),
]);
31 changes: 28 additions & 3 deletions opencti-platform/opencti-graphql/src/schema/stixRefRelationship.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,32 @@
import type { Checker, RelationRefDefinition } from './relationRef-definition';
import { ABSTRACT_STIX_CYBER_OBSERVABLE_RELATIONSHIP, ABSTRACT_STIX_META_RELATIONSHIP, ABSTRACT_STIX_REF_RELATIONSHIP, INPUT_ASSIGNEE, INPUT_BORN_IN, INPUT_CREATED_BY, INPUT_ETHNICITY, INPUT_EXTERNAL_REFS, INPUT_GRANTED_REFS, INPUT_KILLCHAIN, INPUT_LABELS, INPUT_MARKINGS, INPUT_OBJECTS, INPUT_PARTICIPANT } from './general';
import { ENTITY_TYPE_LOCATION_COUNTRY, isStixDomainObjectContainer, isStixDomainObjectIdentity, isStixDomainObjectLocation } from './stixDomainObject';
import { ENTITY_TYPE_EXTERNAL_REFERENCE, ENTITY_TYPE_KILL_CHAIN_PHASE, ENTITY_TYPE_LABEL, ENTITY_TYPE_MARKING_DEFINITION } from './stixMetaObject';
import {
ABSTRACT_STIX_CYBER_OBSERVABLE_RELATIONSHIP,
ABSTRACT_STIX_META_RELATIONSHIP,
ABSTRACT_STIX_REF_RELATIONSHIP,
INPUT_ASSIGNEE,
INPUT_BORN_IN,
INPUT_CREATED_BY,
INPUT_ETHNICITY,
INPUT_EXTERNAL_REFS,
INPUT_GRANTED_REFS,
INPUT_KILLCHAIN,
INPUT_LABELS,
INPUT_MARKINGS,
INPUT_OBJECTS,
INPUT_PARTICIPANT
} from './general';
import {
ENTITY_TYPE_LOCATION_COUNTRY,
isStixDomainObjectContainer,
isStixDomainObjectIdentity,
isStixDomainObjectLocation
} from './stixDomainObject';
import {
ENTITY_TYPE_EXTERNAL_REFERENCE,
ENTITY_TYPE_KILL_CHAIN_PHASE,
ENTITY_TYPE_LABEL,
ENTITY_TYPE_MARKING_DEFINITION
} from './stixMetaObject';
import { ENTITY_TYPE_EVENT } from '../modules/event/event-types';
import { ENTITY_TYPE_USER } from './internalObject';
import { schemaTypesDefinition } from './schema-types';
Expand Down

0 comments on commit 7c59f84

Please sign in to comment.