[api] Deleting entities is not consistent (#935)

opencti-platform/opencti-graphql/src/database/elasticSearch.js

@@ -963,7 +963,8 @@ export const elPaginate = async (indexName, options = {}) => {
       finalSearch = `"*${cleanSearch.replace('https\\://', '')}*"`;
     } else if (cleanSearch.startsWith('"')) {
       finalSearch = `${cleanSearch}`;
-    } else {      const splitSearch = cleanSearch.split(/[\s/]+/);
+    } else {
+      const splitSearch = cleanSearch.split(/[\s/]+/);
       finalSearch = R.pipe(
         R.map((n) => `*${n}*`),
         R.join(' ')

opencti-platform/opencti-graphql/src/database/middleware.js

@@ -1717,8 +1717,11 @@ const getRelatedRelations = async (targetId, elements = [], options = {}) => {
     return { internal_id: internalId, type: entityType, relDependency: true };
   }, connectedRelations);
   elements.push(...connectedRelationsIds);
-  // eslint-disable-next-line no-unused-vars
-  await Promise.all(connectedRelationsIds.map(({ id }) => getRelatedRelations(id, elements, options)));
+  await Promise.all(
+    connectedRelationsIds.map(({ internal_id: id }) => {
+      return getRelatedRelations(id, elements, options);
+    })
+  );
   return elements;
 };
 const getElementsToRemove = async (element, isRelation, options = {}) => {

opencti-platform/opencti-graphql/src/migrations/1608047073975-fix_missing_deletion.js

@@ -0,0 +1,123 @@
+import * as R from 'ramda';
+import { deleteElementByIdRaw, loadById } from '../database/middleware';
+import { ABSTRACT_STIX_META_RELATIONSHIP } from '../schema/general';
+import { DATA_INDICES, el, RELATIONSHIPS_INDICES } from '../database/elasticSearch';
+import { SYSTEM_USER } from '../domain/user';
+
+const average = (arr) => arr.reduce((p, c) => p + c, 0) / arr.length;
+const computeMissingRelationsForType = async (relationType) => {
+  const paginationCount = 500;
+  const relationsToTakeCare = [];
+  let hasNextPage = true;
+  let searchAfter = '';
+  let counter = 0;
+  let timeSpent = 0;
+  const timesSpent = [];
+  let lastRun = new Date();
+  while (hasNextPage) {
+    let body = {
+      size: paginationCount,
+      sort: { 'internal_id.keyword': 'asc' },
+      query: {
+        bool: {
+          should: [{ match_phrase: { entity_type: relationType } }, { match_phrase: { parent_types: relationType } }],
+          minimum_should_match: 1,
+        },
+      },
+    };
+    if (searchAfter) {
+      body = Object.assign(body, { search_after: [searchAfter] });
+    }
+    const query = {
+      index: RELATIONSHIPS_INDICES,
+      _source_includes: ['internal_id', 'entity_type', `connections`],
+      track_total_hits: true,
+      body,
+    };
+    // eslint-disable-next-line no-await-in-loop
+    const queryRelations = await el.search(query);
+    const {
+      hits,
+      total: { value: valTotal },
+    } = queryRelations.body.hits;
+    if (hits.length === 0) {
+      hasNextPage = false;
+      break;
+    }
+    const lastHit = R.last(hits);
+    searchAfter = R.head(lastHit.sort);
+    counter += hits.length;
+    // eslint-disable-next-line no-underscore-dangle
+    const connectionIds = R.uniq(R.flatten(hits.map((h) => h._source.connections.map((c) => c.internal_id))));
+    // eslint-disable-next-line no-await-in-loop
+    const findTerms = connectionIds.map((c) => {
+      return { term: { [`internal_id.keyword`]: c } };
+    });
+    const findQuery = {
+      index: DATA_INDICES,
+      size: 2000,
+      _source_includes: `internal_id`,
+      body: {
+        query: {
+          bool: {
+            should: findTerms,
+            minimum_should_match: 1,
+          },
+        },
+      },
+    };
+    const data = await el.search(findQuery);
+    // eslint-disable-next-line no-underscore-dangle
+    const resolvedConns = data.body.hits.hits.map((i) => i._source);
+    const resolvedIds = resolvedConns.map((r) => r.internal_id);
+    const relationsToRemove = hits
+      // eslint-disable-next-line no-underscore-dangle
+      .map((h) => h._source)
+      .filter((s) => {
+        return !R.all(
+          (a) => resolvedIds.includes(a),
+          s.connections.map((c) => c.internal_id)
+        );
+      });
+    relationsToTakeCare.push(...relationsToRemove);
+    const timeForRun = new Date().getTime() - lastRun.getTime();
+    timesSpent.push(timeForRun);
+    timeSpent += timeForRun;
+    const totalNumberOfIteration = valTotal / paginationCount;
+    const totalEstimation = average(timesSpent) * totalNumberOfIteration;
+    const remaining = (totalEstimation - timeSpent) / 1000 / 60;
+    const findNumber = relationsToTakeCare.length;
+    process.stdout.clearLine();
+    process.stdout.write(
+      `[MIGRATION] Scanning ${relationType}: ${counter}/${valTotal} (Found ${findNumber} to clear) -- Estimate remaining ${remaining.toFixed(
+        2
+      )} min\r`
+    );
+    lastRun = new Date();
+  }
+  return relationsToTakeCare;
+};
+const getMissingRelations = async () => {
+  const data = await computeMissingRelationsForType(ABSTRACT_STIX_META_RELATIONSHIP);
+  return R.flatten(data);
+};
+
+export const up = async (next) => {
+  // Fix missing deleted data
+  // In case of relation to relation, some deletion was not executed.
+  // For each relations of the platform we need to check if the from and the to are available.
+  console.log('[MIGRATION] Starting migration to fix missing deletion');
+  const relations = await getMissingRelations();
+  for (let index = 0; index < relations.length; index += 1) {
+    const relation = relations[index];
+    const element = await loadById(relation.internal_id, relation.entity_type);
+    await deleteElementByIdRaw(SYSTEM_USER, element, element.entity_type);
+  }
+  console.log('\n[MIGRATION] Fix missing deletion migration done');
+  next();
+};
+
+export const down = async (next) => {
+  // Nothing to do.
+  next();
+};