[Frontend/Backend] Skip line when the config give a specific char (#4505)

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperCreation.tsx

@@ -15,6 +15,7 @@ const csvMapperCreation = graphql`
       name
       has_header
       separator
+      skipLineChar
       errors
     }
   }
@@ -39,6 +40,7 @@ const CsvMapperCreation: FunctionComponent<CsvMapperCreationFormProps> = ({
       has_header: values.has_header,
       separator: values.separator,
       representations: JSON.stringify(sanitized(values.representations)),
+      skipLineChar: values.skipLineChar,
     };
     commit({
       variables: {
@@ -64,6 +66,7 @@ const CsvMapperCreation: FunctionComponent<CsvMapperCreationFormProps> = ({
     has_header: false,
     separator: ',',
     representations: [],
+    skipLineChar: '',
     errors: null,
   };
 

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperEdition.tsx

@@ -34,6 +34,7 @@ const CsvMapperEdition: FunctionComponent<CsvMapperEditionProps> = ({
     name: csvMapper.name,
     has_header: csvMapper.has_header,
     separator: csvMapper.separator,
+    skipLineChar: csvMapper.skipLineChar,
     representations: useMapRepresentations(csvMapper.representations),
     errors: csvMapper.errors,
   };

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperEditionContainer.tsx

@@ -6,8 +6,12 @@ import {
   usePreloadedQuery,
 } from 'react-relay';
 import CsvMapperEdition from '@components/data/csvMapper/CsvMapperEdition';
-import { CsvMapperEditionContainerFragment_csvMapper$key } from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerFragment_csvMapper.graphql';
-import { CsvMapperEditionContainerQuery } from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerQuery.graphql';
+import {
+  CsvMapperEditionContainerFragment_csvMapper$key,
+} from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerFragment_csvMapper.graphql';
+import {
+  CsvMapperEditionContainerQuery,
+} from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerQuery.graphql';
 import Drawer from '@components/common/drawer/Drawer';
 import Loader, { LoaderVariant } from '../../../../components/Loader';
 import { useFormatter } from '../../../../components/i18n';
@@ -18,6 +22,7 @@ const csvMapperEditionContainerFragment = graphql`
     name
     has_header
     separator
+    skipLineChar
     errors
     representations {
       id
@@ -70,12 +75,12 @@ const CsvMapperEditionContainer: FunctionComponent<CsvMapperEditionProps> = ({
   );
 
   if (!csvMapper) {
-    return <Loader variant={LoaderVariant.inElement} />;
+    return <Loader variant={LoaderVariant.inElement}/>;
   }
 
   return (
     <Drawer title={t('Csv Mapper edition')} open={open} onClose={onClose}>
-      <CsvMapperEdition csvMapper={csvMapper} onClose={onClose} />
+      <CsvMapperEdition csvMapper={csvMapper} onClose={onClose}/>
     </Drawer>
   );
 };

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperForm.tsx

@@ -1,6 +1,5 @@
 import React, { FunctionComponent, useEffect, useState } from 'react';
 import { Field, Form, Formik } from 'formik';
-import { TextField } from 'formik-mui';
 import Button from '@mui/material/Button';
 import makeStyles from '@mui/styles/makeStyles';
 import * as Yup from 'yup';
@@ -14,6 +13,8 @@ import CsvMapperRepresentationForm, {
   RepresentationFormEntityOption,
 } from '@components/data/csvMapper/representations/CsvMapperRepresentationForm';
 import { CsvMapper } from '@components/data/csvMapper/CsvMapper';
+import TextField from 'src/components/TextField';
+import classNames from 'classnames';
 import { Theme } from '../../../../components/Theme';
 import { useFormatter } from '../../../../components/i18n';
 import SwitchField from '../../../../components/SwitchField';
@@ -34,12 +35,23 @@ const useStyles = makeStyles<Theme>((theme) => ({
     display: 'flex',
     alignItems: 'center',
   },
+  marginTop: {
+    marginTop: 20,
+  },
+  formContainer: {
+    margin: '20px 0',
+  },
+  representationContainer: {
+    marginTop: 20,
+    display: 'flex',
+  },
 }));
 
 const csvMapperValidation = (t: (s: string) => string) => Yup.object().shape({
   name: Yup.string().required(t('This field is required')),
   has_header: Yup.boolean().required(t('This field is required')),
   separator: Yup.string().required(t('This field is required')),
+  skipLineChar: Yup.string().max(1),
 });
 
 interface CsvMapperFormProps {
@@ -50,10 +62,7 @@ interface CsvMapperFormProps {
   ) => void;
 }
 
-const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
-  csvMapper,
-  onSubmit,
-}) => {
+const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({ csvMapper, onSubmit }) => {
   const { t } = useFormatter();
   const classes = useStyles();
 
@@ -157,15 +166,15 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
           );
 
           return (
-            <Form style={{ margin: '20px 0 20px 0' }}>
+            <Form className={classes.formContainer}>
               <Field
                 component={TextField}
                 variant="standard"
                 name="name"
                 label={t('Name')}
                 fullWidth
               />
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={classNames(classes.center, classes.marginTop)}>
                 <Field
                   component={SwitchField}
                   type="checkbox"
@@ -184,7 +193,7 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   />
                 </Tooltip>
               </div>
-              <div style={{ marginTop: 20 }}>
+              <div className={classes.marginTop}>
                 <Typography>{t('CSV separator')}</Typography>
                 <div className={classes.center}>
                   <Field
@@ -213,11 +222,30 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   <Typography>{t('Semicolon')}</Typography>
                 </div>
               </div>
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={classes.center}>
+                <Field
+                  component={TextField}
+                  name="skipLineChar"
+                  value={values.skipLineChar}
+                  label={t('Char to escape line')}
+                  onChange={(event: SelectChangeEvent) => setFieldValue('skipLineChar', event.target.value)}
+                />
+                <Tooltip
+                  title={t(
+                    'Every line that begins with this character will be skipped during parsing (for example: #).',
+                  )}
+                >
+                  <InformationOutline
+                    fontSize="small"
+                    color="primary"
+                    style={{ cursor: 'default' }}
+                  />
+                </Tooltip>
+              </div>
+              <div className={classNames(classes.center, classes.marginTop)}>
                 <Typography
                   variant="h3"
                   gutterBottom
-                  style={{ marginBottom: 0 }}
                 >
                   {t('Representations for entity')}
                 </Typography>
@@ -228,13 +256,13 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   }
                   size="large"
                 >
-                  <Add fontSize="small" />
+                  <Add fontSize="small"/>
                 </IconButton>
               </div>
               {entities.map((representation, idx) => (
                 <div
                   key={`entity-${idx}`}
-                  style={{ marginTop: 20, display: 'flex' }}
+                  className={classes.representationContainer}
                 >
                   <CsvMapperRepresentationForm
                     key={representation.id}
@@ -245,11 +273,10 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   />
                 </div>
               ))}
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={classNames(classes.center, classes.marginTop)}>
                 <Typography
                   variant="h3"
                   gutterBottom
-                  style={{ marginBottom: 0 }}
                 >
                   {t('Representations for relationship')}
                 </Typography>
@@ -260,13 +287,13 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   }
                   size="large"
                 >
-                  <Add fontSize="small" />
+                  <Add fontSize="small"/>
                 </IconButton>
               </div>
               {relationships.map((representation, idx) => (
                 <div
                   key={`relationship-${idx}`}
-                  style={{ marginTop: 20, display: 'flex' }}
+                  className={classes.representationContainer}
                 >
                   <CsvMapperRepresentationForm
                     key={representation.id}

opencti-platform/opencti-front/src/schema/relay.schema.graphql

@@ -12403,6 +12403,7 @@ type CsvMapper implements InternalObject & BasicObject {
   name: String!
   has_header: Boolean!
   separator: String!
+  skipLineChar: String
   representations: [CsvMapperRepresentation!]!
   errors: String
 }
@@ -12491,4 +12492,5 @@ input CsvMapperAddInput {
   has_header: Boolean!
   separator: String!
   representations: String!
+  skipLineChar: String
 }

opencti-platform/opencti-front/src/utils/Localization.js

@@ -2210,6 +2210,7 @@ const i18n = {
       'Grantable groups by organization administrators': 'Grupos autorizables por los administradores de la organización',
       'This Group allows the user to bypass restriction. It should not be added here.': 'Este grupo permite al usuario saltarse las restricciones. No debe añadirse aquí',
       'Add a group': 'Añadir un grupo',
+      'Char to escape line': 'Carácter para escapar la línea',
       Processing: 'Procesamiento',
       Automation: 'Automatización',
       'My CSV file contains headers': 'Mi archivo CSV contiene encabezados',
@@ -4432,6 +4433,7 @@ const i18n = {
       'Grantable groups by organization administrators': 'Groupes autorisés par les administrateurs d\'organisations',
       'This Group allows the user to bypass restriction. It should not be added here.': 'Ce groupe permet à l\'utilisateur de contourner les restrictions. Il ne doit pas être ajouté ici',
       'Add a group': 'Ajouter un groupe',
+      'Char to escape line': 'Caractère pour sauter une ligne',
       Processing: 'Traitement',
       Automation: 'Automatisation',
       'My CSV file contains headers': 'Mon fichier CSV contient des en-têtes',
@@ -6573,6 +6575,7 @@ const i18n = {
       'Grantable groups by organization administrators': '組織管理者が付与可能なグループ',
       'This Group allows the user to bypass restriction. It should not be added here.': 'このグループは、ユーザーが制限をバイパスすることができます。ここには追加しないでください、',
       'Add a group': 'グループを追加します',
+      'Char to escape line': 'ラインをエスケープするための文字',
       Processing: '処理',
       Automation: '自動化',
       'My CSV file contains headers': 'CSVファイルにヘッダーが含まれています',
@@ -8610,6 +8613,7 @@ const i18n = {
       'Grantable groups by organization administrators': '组织管理员可授予的组',
       'This Group allows the user to bypass restriction. It should not be added here.': '该组允许用户绕过限制。不应在此添加',
       'Add a group': '添加组',
+      'Char to escape line': 'ラインをエスケープするための文字',
       Processing: '处理',
       Automation: '自动化',
       'My CSV file contains headers': '我的CSV文件包含标题',

opencti-platform/opencti-graphql/src/generated/graphql.ts

@@ -4974,6 +4974,7 @@ export type CsvMapper = BasicObject & InternalObject & {
   parent_types: Array<Scalars['String']['output']>;
   representations: Array<CsvMapperRepresentation>;
   separator: Scalars['String']['output'];
+  skipLineChar?: Maybe<Scalars['String']['output']>;
   standard_id: Scalars['String']['output'];
 };
 
@@ -4982,6 +4983,7 @@ export type CsvMapperAddInput = {
   name: Scalars['String']['input'];
   representations: Scalars['String']['input'];
   separator: Scalars['String']['input'];
+  skipLineChar?: InputMaybe<Scalars['String']['input']>;
 };
 
 export type CsvMapperConnection = {
@@ -32223,6 +32225,7 @@ export type CsvMapperResolvers<ContextType = any, ParentType extends ResolversPa
   parent_types?: Resolver<Array<ResolversTypes['String']>, ParentType, ContextType>;
   representations?: Resolver<Array<ResolversTypes['CsvMapperRepresentation']>, ParentType, ContextType>;
   separator?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  skipLineChar?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   standard_id?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 }>;

opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper-types.ts

@@ -57,6 +57,7 @@ export interface BasicStoreEntityCsvMapper extends BasicStoreEntity {
   name: string
   has_header: boolean
   separator: string
+  skipLineChar: string
   representations: CsvMapperRepresentation[]
 }
 

opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper.graphql

@@ -60,6 +60,7 @@ type CsvMapper implements InternalObject & BasicObject {
   name: String! @auth
   has_header: Boolean! @auth
   separator: String! @auth
+  skipLineChar: String @auth
   representations: [CsvMapperRepresentation!]! @auth
   errors: String
 }
@@ -163,6 +164,7 @@ input CsvMapperAddInput {
   has_header: Boolean!
   separator: String!
   representations: String!
+  skipLineChar: String
 }
 
 type Mutation {

opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper.ts

@@ -34,6 +34,7 @@ const CSV_MAPPER_DEFINITION: ModuleDefinition<StoreEntityCsvMapper, StixCsvMappe
     { name: 'has_header', type: 'boolean', mandatoryType: 'internal', multiple: false, upsert: false },
     { name: 'separator', type: 'string', mandatoryType: 'internal', multiple: false, upsert: false },
     { name: 'representations', type: 'json', mandatoryType: 'internal', multiple: false, upsert: false },
+    { name: 'skipLineChar', type: 'string', mandatoryType: 'no', multiple: false, upsert: false },
   ],
   relations: [],
   representative: (instance: StixCsvMapper) => {

opencti-platform/opencti-graphql/src/parser/csv-bundler.ts

@@ -32,8 +32,10 @@ export const bundleProcess = async (context: AuthContext, user: AuthUser, conten
 
   const bundleBuilder = new BundleBuilder();
   let skipLine = sanitizedMapper.has_header;
-
-  const records = await parsingProcess(content, mapper.separator);
+  let records = await parsingProcess(content, mapper.separator);
+  if (mapper.skipLineChar) {
+    records = records.filter((record) => !record[0].startsWith(mapper.skipLineChar));
+  }
   if (records) {
     await Promise.all((records.map(async (record: string[]) => {
       const isEmptyLine = record.length === 1 && isEmptyField(record[0]);

opencti-platform/opencti-graphql/tests/02-integration/05-parser/csv-parser-test.js

@@ -13,6 +13,7 @@ import {
 import { csvMapperMockSimpleSighting } from "./simple-sighting-test/csv-mapper-mock-simple-sighting";
 import { bundleProcess } from "../../../src/parser/csv-bundler";
 import { ADMIN_USER, testContext } from "../../utils/testQuery";
+import { csvMapperMockSimpleSkipLine } from "./simple-skip-line-test/csv-mapper-mock-simple-skip-line";
 
 describe('CSV-HELPER', () => {
   it('Column name to idx', async () => {
@@ -144,4 +145,19 @@ describe('CSV-PARSER', () => {
     expect(relationshipPartOf.length)
       .toBe(160);
   });
+  it('Parse CSV - Simple skip line test on Simple entity ', async () => {
+    const filPath = './tests/02-integration/05-parser/simple-skip-line-test/Threat-Actor-Group_list_skip_line.csv';
+    const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleSkipLine);
+    const objects = bundle.objects;
+    expect(objects.length)
+      .toBe(5);
+    expect(objects.filter((o) => isNotEmptyField(o.name)).length)
+      .toBe(5);
+    const threatActorWithTypes = objects.filter((o) => isNotEmptyField(o.threat_actor_types))[0];
+    expect(threatActorWithTypes)
+      .not
+      .toBeNull();
+    expect(threatActorWithTypes.threat_actor_types.length)
+      .toBe(2);
+  });
 })

opencti-platform/opencti-graphql/tests/02-integration/05-parser/simple-skip-line-test/Threat-Actor-Group_list_skip_line.csv

@@ -0,0 +1,13 @@
+################################################################
+# Test (CSV)                                                   #
+# Last updated: 2023-10-11 19:27:44 UTC                        #
+#                                                              #
+################################################################
+#
+"aliases";"confidence";"created";"createdBy";"createdById";"created_at";"description";"entity_type";"externalReferences";"externalReferencesIds";"first_seen";"goals";"id";"importFiles";"importFilesIds";"last_seen";"modified";"name";"objectLabel";"objectLabelIds";"objectMarking";"objectMarkingIds";"parent_types";"personal_motivations";"primary_motivation";"resource_level";"revoked";"roles";"secondary_motivations";"sophistication";"spec_version";"standard_id";"threat_actor_types";"updated_at"
+"";"75";"2023-08-21T08:42:49.984Z";"";"";"2023-08-21T08:42:49.984Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"f201692e-81e7-4d0a-bdfd-c2196c952259";"WhatsApp Image 2023-07-08 à 20.46.05.jpg";"import/Threat-Actor-Group/f201692e-81e7-4d0a-bdfd-c2196c952259/WhatsApp Image 2023-07-08 à 20.46.05.jpg";"5138-11-16T09:46:40.000Z";"2023-08-21T08:42:49.984Z";"A threat actor group";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--22d840e1-9c6a-58ef-aafe-383d39357113";"";"2023-08-30T08:14:28.042Z"
+"";"90";"2023-06-30T08:53:38.288Z";"Russie";"0d5c41e2-55ab-400c-af8d-7dc40a53cd0a";"2023-08-28T13:26:59.245Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"214c000e-6c89-4638-babd-8bcf1eb9a0ae";"";"";"5138-11-16T09:46:40.000Z";"2023-08-28T13:26:59.386Z";"RRN";"russia,rrn";"e4ac327e-b641-4cb2-af77-de6bde2507a6,c70a11ff-5517-4487-9471-eb7879dfc32a";"TLP:CLEAR";"112fd0bd-2d5e-453e-8138-621022be29c0";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--5c18e55e-2244-5a24-980e-f8fa658878f7";nation-state, activist;"2023-08-28T13:26:59.386Z"
+"";"75";"2023-08-22T14:52:17.416Z";"";"";"2023-08-22T14:52:17.416Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"f2eb14cc-a311-4b97-a197-d66c5444a64a";"";"";"5138-11-16T09:46:40.000Z";"2023-08-22T14:52:17.416Z";"TAG Test 3";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--3a490a21-0f0b-5000-97e8-1e377ad25b6a";"";"2023-08-22T14:52:17.416Z"
+"";"75";"2023-08-22T14:31:17.371Z";"";"";"2023-08-22T14:31:17.371Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"301e4271-73cc-41b9-a257-4fe8b6cc2d10";"";"";"5138-11-16T09:46:40.000Z";"2023-08-22T14:31:17.371Z";"TAG2";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--1749c7f4-049a-5f12-9de4-48c50f52bdb3";"";"2023-08-22T14:31:17.371Z"
+"";"75";"2023-08-31T08:35:57.033Z";"";"";"2023-08-31T08:35:57.033Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"302047b3-017e-4127-83be-c28e13d54700";"";"";"5138-11-16T09:46:40.000Z";"2023-08-31T08:35:57.033Z";"Threat actor group test";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--f57fa645-6a47-535f-9015-8f5602878437";"";"2023-08-31T08:36:53.873Z"
+# END 5 entries