[Frontend/Backend] Skip line when the config give a specific char

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperCreation.tsx

@@ -9,15 +9,16 @@ import { CsvMapperAddInput } from '@components/data/csvMapper/__generated__/CsvM
 import { insertNode } from '../../../../utils/store';
 
 const csvMapperCreation = graphql`
-  mutation CsvMapperCreationContainerMutation($input: CsvMapperAddInput!) {
-    csvMapperAdd(input: $input) {
-      id
-      name
-      has_header
-      separator
-      errors
+    mutation CsvMapperCreationContainerMutation($input: CsvMapperAddInput!) {
+        csvMapperAdd(input: $input) {
+            id
+            name
+            has_header
+            separator
+            skipLineChar
+            errors
+        }
     }
-  }
 `;
 
 interface CsvMapperCreationFormProps {
@@ -39,6 +40,7 @@ const CsvMapperCreation: FunctionComponent<CsvMapperCreationFormProps> = ({
       has_header: values.has_header,
       separator: values.separator,
       representations: JSON.stringify(sanitized(values.representations)),
+      skipLineChar: values.skipLineChar,
     };
     commit({
       variables: {
@@ -64,6 +66,7 @@ const CsvMapperCreation: FunctionComponent<CsvMapperCreationFormProps> = ({
     has_header: false,
     separator: ',',
     representations: [],
+    skipLineChar: '',
     errors: null,
   };
 

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperEdition.tsx

@@ -34,6 +34,7 @@ const CsvMapperEdition: FunctionComponent<CsvMapperEditionProps> = ({
     name: csvMapper.name,
     has_header: csvMapper.has_header,
     separator: csvMapper.separator,
+    skipLineChar: csvMapper?.skipLineChar,
     representations: useMapRepresentations(csvMapper.representations),
     errors: csvMapper.errors,
   };

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperEditionContainer.tsx

@@ -13,33 +13,34 @@ import Loader, { LoaderVariant } from '../../../../components/Loader';
 import { useFormatter } from '../../../../components/i18n';
 
 const csvMapperEditionContainerFragment = graphql`
-  fragment CsvMapperEditionContainerFragment_csvMapper on CsvMapper {
-    id
-    name
-    has_header
-    separator
-    errors
-    representations {
-      id
-      type
-      target {
-        entity_type
-      }
-      attributes {
-        key
-        column {
-          column_name
-          configuration {
-            separator
-            pattern_date
-          }
+    fragment CsvMapperEditionContainerFragment_csvMapper on CsvMapper {
+        id
+        name
+        has_header
+        separator
+        skipLineChar
+        errors
+        representations {
+            id
+            type
+            target {
+                entity_type
+            }
+            attributes {
+                key
+                column {
+                    column_name
+                    configuration {
+                        separator
+                        pattern_date
+                    }
+                }
+                based_on {
+                    representations
+                }
+            }
         }
-        based_on {
-          representations
-        }
-      }
     }
-  }
 `;
 
 export const csvMapperEditionContainerQuery = graphql`

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperForm.tsx

@@ -34,12 +34,23 @@ const useStyles = makeStyles<Theme>((theme) => ({
     display: 'flex',
     alignItems: 'center',
   },
+  marginTop: {
+    marginTop: 20,
+  },
+  formContainer: {
+    margin: '20px 0',
+  },
+  representationContainer: {
+    marginTop: 20,
+    display: 'flex',
+  },
 }));
 
 const csvMapperValidation = (t: (s: string) => string) => Yup.object().shape({
   name: Yup.string().required(t('This field is required')),
   has_header: Yup.boolean().required(t('This field is required')),
   separator: Yup.string().required(t('This field is required')),
+  skipLineChar: Yup.string(),
 });
 
 interface CsvMapperFormProps {
@@ -157,15 +168,15 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
           );
 
           return (
-            <Form style={{ margin: '20px 0 20px 0' }}>
+            <Form className={classes.formContainer}>
               <Field
                 component={TextField}
                 variant="standard"
                 name="name"
                 label={t('Name')}
                 fullWidth
               />
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={`${classes.center} ${classes.marginTop}`}>
                 <Field
                   component={SwitchField}
                   type="checkbox"
@@ -184,7 +195,7 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   />
                 </Tooltip>
               </div>
-              <div style={{ marginTop: 20 }}>
+              <div className={classes.marginTop}>
                 <Typography>{t('CSV separator')}</Typography>
                 <div className={classes.center}>
                   <Field
@@ -213,11 +224,19 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   <Typography>{t('Semicolon')}</Typography>
                 </div>
               </div>
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={`${classes.center} ${classes.marginTop}`}>
+                  <Field
+                    component={TextField}
+                    name="skipLineChar"
+                    value={values.skipLineChar}
+                    label={t('Char to escape line')}
+                    onChange={(event: SelectChangeEvent) => setFieldValue('skipLineChar', event.target.value)}
+                  />
+              </div>
+              <div className={`${classes.center} ${classes.marginTop}`}>
                 <Typography
                   variant="h3"
                   gutterBottom
-                  style={{ marginBottom: 0 }}
                 >
                   {t('Representations for entity')}
                 </Typography>
@@ -234,7 +253,7 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
               {entities.map((representation, idx) => (
                 <div
                   key={`entity-${idx}`}
-                  style={{ marginTop: 20, display: 'flex' }}
+                  className={classes.representationContainer}
                 >
                   <CsvMapperRepresentationForm
                     key={representation.id}
@@ -245,11 +264,10 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   />
                 </div>
               ))}
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={`${classes.center} ${classes.marginTop}`}>
                 <Typography
                   variant="h3"
                   gutterBottom
-                  style={{ marginBottom: 0 }}
                 >
                   {t('Representations for relationship')}
                 </Typography>
@@ -266,7 +284,7 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
               {relationships.map((representation, idx) => (
                 <div
                   key={`relationship-${idx}`}
-                  style={{ marginTop: 20, display: 'flex' }}
+                  className={classes.representationContainer}
                 >
                   <CsvMapperRepresentationForm
                     key={representation.id}

opencti-platform/opencti-front/src/schema/relay.schema.graphql

@@ -12352,6 +12352,7 @@ type CsvMapper implements InternalObject & BasicObject {
   name: String!
   has_header: Boolean!
   separator: String!
+  skipLineChar: String
   representations: [CsvMapperRepresentation!]!
   errors: String
 }
@@ -12434,4 +12435,5 @@ input CsvMapperAddInput {
   has_header: Boolean!
   separator: String!
   representations: String!
+  skipLineChar: String
 }

opencti-platform/opencti-front/src/utils/Localization.js

@@ -2183,6 +2183,7 @@ const i18n = {
       'Grantable groups by organization administrators': 'Grupos autorizables por los administradores de la organización',
       'This Group allows the user to bypass restriction. It should not be added here.': 'Este grupo permite al usuario saltarse las restricciones. No debe añadirse aquí',
       'Add a group': 'Añadir un grupo',
+      'Char to escape line': 'Carácter para escapar la línea',
     },
     'fr-fr': {
       // Titles
@@ -4371,6 +4372,7 @@ const i18n = {
       'Grantable groups by organization administrators': 'Groupes autorisés par les administrateurs d\'organisations',
       'This Group allows the user to bypass restriction. It should not be added here.': 'Ce groupe permet à l\'utilisateur de contourner les restrictions. Il ne doit pas être ajouté ici',
       'Add a group': 'Ajouter un groupe',
+      'Char to escape line': 'Caractère pour sauter une ligne',
     },
     'ja-jp': {
       // Titles
@@ -6479,6 +6481,7 @@ const i18n = {
       'Grantable groups by organization administrators': '組織管理者が付与可能なグループ',
       'This Group allows the user to bypass restriction. It should not be added here.': 'このグループは、ユーザーが制限をバイパスすることができます。ここには追加しないでください、',
       'Add a group': 'グループを追加します',
+      'Char to escape line': 'ラインをエスケープするための文字',
     },
     'zh-cn': {
       // Titles
@@ -8485,6 +8488,7 @@ const i18n = {
       'Grantable groups by organization administrators': '组织管理员可授予的组',
       'This Group allows the user to bypass restriction. It should not be added here.': '该组允许用户绕过限制。不应在此添加',
       'Add a group': '添加组',
+      'Char to escape line': '用于转义行的字符',
     },
     'en-us': {
       gt: 'Greater than',

opencti-platform/opencti-graphql/src/generated/graphql.ts

@@ -4967,6 +4967,7 @@ export type CsvMapper = BasicObject & InternalObject & {
   parent_types: Array<Scalars['String']['output']>;
   representations: Array<CsvMapperRepresentation>;
   separator: Scalars['String']['output'];
+  skipLineChar?: Maybe<Scalars['String']['output']>;
   standard_id: Scalars['String']['output'];
 };
 
@@ -4975,6 +4976,7 @@ export type CsvMapperAddInput = {
   name: Scalars['String']['input'];
   representations: Scalars['String']['input'];
   separator: Scalars['String']['input'];
+  skipLineChar?: InputMaybe<Scalars['String']['input']>;
 };
 
 export type CsvMapperConnection = {
@@ -32108,6 +32110,7 @@ export type CsvMapperResolvers<ContextType = any, ParentType extends ResolversPa
   parent_types?: Resolver<Array<ResolversTypes['String']>, ParentType, ContextType>;
   representations?: Resolver<Array<ResolversTypes['CsvMapperRepresentation']>, ParentType, ContextType>;
   separator?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  skipLineChar?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   standard_id?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
 }>;

opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper-types.ts

@@ -57,6 +57,7 @@ export interface BasicStoreEntityCsvMapper extends BasicStoreEntity {
   name: string
   has_header: boolean
   separator: string
+  skipLineChar: string
   representations: CsvMapperRepresentation[]
 }
 

opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper.graphql

@@ -60,6 +60,7 @@ type CsvMapper implements InternalObject & BasicObject {
   name: String! @auth
   has_header: Boolean! @auth
   separator: String! @auth
+  skipLineChar: String @auth
   representations: [CsvMapperRepresentation!]! @auth
   errors: String
 }
@@ -157,6 +158,7 @@ input CsvMapperAddInput {
   has_header: Boolean!
   separator: String!
   representations: String!
+  skipLineChar: String
 }
 
 type Mutation {

opencti-platform/opencti-graphql/src/modules/internal/csvMapper/csvMapper.ts

@@ -34,6 +34,7 @@ const CSV_MAPPER_DEFINITION: ModuleDefinition<StoreEntityCsvMapper, StixCsvMappe
     { name: 'has_header', type: 'boolean', mandatoryType: 'internal', multiple: false, upsert: false },
     { name: 'separator', type: 'string', mandatoryType: 'internal', multiple: false, upsert: false },
     { name: 'representations', type: 'json', mandatoryType: 'internal', multiple: false, upsert: false },
+    { name: 'skipLineChar', type: 'string', mandatoryType: 'no', multiple: false, upsert: false },
   ],
   relations: [],
   representative: (instance: StixCsvMapper) => {

opencti-platform/opencti-graphql/src/parser/csv-bundler.ts

@@ -32,8 +32,10 @@ export const bundleProcess = async (context: AuthContext, user: AuthUser, conten
 
   const bundleBuilder = new BundleBuilder();
   let skipLine = sanitizedMapper.has_header;
-
-  const records = await parsingProcess(content, mapper.separator);
+  let records = await parsingProcess(content, mapper.separator);
+  if (mapper.skipLineChar) {
+    records = records.filter((record) => !record[0].startsWith(mapper.skipLineChar));
+  }
   if (records) {
     await Promise.all((records.map(async (record: string[]) => {
       const isEmptyLine = record.length === 1 && isEmptyField(record[0]);

opencti-platform/opencti-graphql/tests/02-integration/05-parser/csv-parser-test.js

@@ -13,6 +13,7 @@ import {
 import { csvMapperMockSimpleSighting } from "./simple-sighting-test/csv-mapper-mock-simple-sighting";
 import { bundleProcess } from "../../../src/parser/csv-bundler";
 import { ADMIN_USER, testContext } from "../../utils/testQuery";
+import { csvMapperMockSimpleSkipLine } from "./simple-skip-line-test/csv-mapper-mock-simple-skip-line";
 
 describe('CSV-HELPER', () => {
   it('Column name to idx', async () => {
@@ -144,4 +145,19 @@ describe('CSV-PARSER', () => {
     expect(relationshipPartOf.length)
       .toBe(160);
   });
+  it('Parse CSV - Simple skip line test on Simple entity ', async () => {
+    const filPath = './tests/02-integration/05-parser/simple-skip-line-test/Threat-Actor-Group_list_skip_line.csv';
+    const bundle = await bundleProcess(testContext, ADMIN_USER, filPath, csvMapperMockSimpleSkipLine);
+    const objects = bundle.objects;
+    expect(objects.length)
+      .toBe(5);
+    expect(objects.filter((o) => isNotEmptyField(o.name)).length)
+      .toBe(5);
+    const threatActorWithTypes = objects.filter((o) => isNotEmptyField(o.threat_actor_types))[0];
+    expect(threatActorWithTypes)
+      .not
+      .toBeNull();
+    expect(threatActorWithTypes.threat_actor_types.length)
+      .toBe(2);
+  });
 })

opencti-platform/opencti-graphql/tests/02-integration/05-parser/simple-skip-line-test/Threat-Actor-Group_list_skip_line.csv

@@ -0,0 +1,13 @@
+################################################################
+# Test (CSV)                                                   #
+# Last updated: 2023-10-11 19:27:44 UTC                        #
+#                                                              #
+################################################################
+#
+"aliases";"confidence";"created";"createdBy";"createdById";"created_at";"description";"entity_type";"externalReferences";"externalReferencesIds";"first_seen";"goals";"id";"importFiles";"importFilesIds";"last_seen";"modified";"name";"objectLabel";"objectLabelIds";"objectMarking";"objectMarkingIds";"parent_types";"personal_motivations";"primary_motivation";"resource_level";"revoked";"roles";"secondary_motivations";"sophistication";"spec_version";"standard_id";"threat_actor_types";"updated_at"
+"";"75";"2023-08-21T08:42:49.984Z";"";"";"2023-08-21T08:42:49.984Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"f201692e-81e7-4d0a-bdfd-c2196c952259";"WhatsApp Image 2023-07-08 à 20.46.05.jpg";"import/Threat-Actor-Group/f201692e-81e7-4d0a-bdfd-c2196c952259/WhatsApp Image 2023-07-08 à 20.46.05.jpg";"5138-11-16T09:46:40.000Z";"2023-08-21T08:42:49.984Z";"A threat actor group";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--22d840e1-9c6a-58ef-aafe-383d39357113";"";"2023-08-30T08:14:28.042Z"
+"";"90";"2023-06-30T08:53:38.288Z";"Russie";"0d5c41e2-55ab-400c-af8d-7dc40a53cd0a";"2023-08-28T13:26:59.245Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"214c000e-6c89-4638-babd-8bcf1eb9a0ae";"";"";"5138-11-16T09:46:40.000Z";"2023-08-28T13:26:59.386Z";"RRN";"russia,rrn";"e4ac327e-b641-4cb2-af77-de6bde2507a6,c70a11ff-5517-4487-9471-eb7879dfc32a";"TLP:CLEAR";"112fd0bd-2d5e-453e-8138-621022be29c0";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--5c18e55e-2244-5a24-980e-f8fa658878f7";nation-state, activist;"2023-08-28T13:26:59.386Z"
+"";"75";"2023-08-22T14:52:17.416Z";"";"";"2023-08-22T14:52:17.416Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"f2eb14cc-a311-4b97-a197-d66c5444a64a";"";"";"5138-11-16T09:46:40.000Z";"2023-08-22T14:52:17.416Z";"TAG Test 3";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--3a490a21-0f0b-5000-97e8-1e377ad25b6a";"";"2023-08-22T14:52:17.416Z"
+"";"75";"2023-08-22T14:31:17.371Z";"";"";"2023-08-22T14:31:17.371Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"301e4271-73cc-41b9-a257-4fe8b6cc2d10";"";"";"5138-11-16T09:46:40.000Z";"2023-08-22T14:31:17.371Z";"TAG2";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--1749c7f4-049a-5f12-9de4-48c50f52bdb3";"";"2023-08-22T14:31:17.371Z"
+"";"75";"2023-08-31T08:35:57.033Z";"";"";"2023-08-31T08:35:57.033Z";"";"Threat-Actor-Group";"";"";1970-01-01;"";"302047b3-017e-4127-83be-c28e13d54700";"";"";"5138-11-16T09:46:40.000Z";"2023-08-31T08:35:57.033Z";"Threat actor group test";"";"";"";"";"Basic-Object,Stix-Object,Stix-Core-Object,Stix-Domain-Object,Threat-Actor";"";"";"";"False";"";"";"";"2.1";"threat-actor--f57fa645-6a47-535f-9015-8f5602878437";"";"2023-08-31T08:36:53.873Z"
+# END 5 entries