[Frontend/Backend] Skip line when the config give a specific char

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperCreation.tsx

@@ -15,6 +15,7 @@ const csvMapperCreation = graphql`
       name
       has_header
       separator
+      skipLineChar
       errors
     }
   }
@@ -39,6 +40,7 @@ const CsvMapperCreation: FunctionComponent<CsvMapperCreationFormProps> = ({
       has_header: values.has_header,
       separator: values.separator,
       representations: JSON.stringify(sanitized(values.representations)),
+      skipLineChar: values.skipLineChar,
     };
     commit({
       variables: {
@@ -64,6 +66,7 @@ const CsvMapperCreation: FunctionComponent<CsvMapperCreationFormProps> = ({
     has_header: false,
     separator: ',',
     representations: [],
+    skipLineChar: '',
     errors: null,
   };
 

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperEdition.tsx

@@ -34,6 +34,7 @@ const CsvMapperEdition: FunctionComponent<CsvMapperEditionProps> = ({
     name: csvMapper.name,
     has_header: csvMapper.has_header,
     separator: csvMapper.separator,
+    skipLineChar: csvMapper.skipLineChar,
     representations: useMapRepresentations(csvMapper.representations),
     errors: csvMapper.errors,
   };

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperEditionContainer.tsx

@@ -6,8 +6,12 @@ import {
   usePreloadedQuery,
 } from 'react-relay';
 import CsvMapperEdition from '@components/data/csvMapper/CsvMapperEdition';
-import { CsvMapperEditionContainerFragment_csvMapper$key } from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerFragment_csvMapper.graphql';
-import { CsvMapperEditionContainerQuery } from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerQuery.graphql';
+import {
+  CsvMapperEditionContainerFragment_csvMapper$key,
+} from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerFragment_csvMapper.graphql';
+import {
+  CsvMapperEditionContainerQuery,
+} from '@components/data/csvMapper/__generated__/CsvMapperEditionContainerQuery.graphql';
 import Drawer from '@components/common/drawer/Drawer';
 import Loader, { LoaderVariant } from '../../../../components/Loader';
 import { useFormatter } from '../../../../components/i18n';
@@ -18,6 +22,7 @@ const csvMapperEditionContainerFragment = graphql`
     name
     has_header
     separator
+    skipLineChar
     errors
     representations {
       id
@@ -70,12 +75,12 @@ const CsvMapperEditionContainer: FunctionComponent<CsvMapperEditionProps> = ({
   );
 
   if (!csvMapper) {
-    return <Loader variant={LoaderVariant.inElement} />;
+    return <Loader variant={LoaderVariant.inElement}/>;
   }
 
   return (
     <Drawer title={t('Csv Mapper edition')} open={open} onClose={onClose}>
-      <CsvMapperEdition csvMapper={csvMapper} onClose={onClose} />
+      <CsvMapperEdition csvMapper={csvMapper} onClose={onClose}/>
     </Drawer>
   );
 };

opencti-platform/opencti-front/src/private/components/data/csvMapper/CsvMapperForm.tsx

@@ -1,26 +1,27 @@
-import React, { FunctionComponent, useEffect, useState } from 'react';
-import { Field, Form, Formik } from 'formik';
-import { TextField } from 'formik-mui';
+import React, {FunctionComponent, useEffect, useState} from 'react';
+import {Field, Form, Formik} from 'formik';
 import Button from '@mui/material/Button';
 import makeStyles from '@mui/styles/makeStyles';
 import * as Yup from 'yup';
-import { IconButton, Radio, Typography } from '@mui/material';
-import { Add } from '@mui/icons-material';
-import { InformationOutline } from 'mdi-material-ui';
+import {IconButton, Radio, Typography} from '@mui/material';
+import {Add} from '@mui/icons-material';
+import {InformationOutline} from 'mdi-material-ui';
 import Tooltip from '@mui/material/Tooltip';
-import { FormikHelpers } from 'formik/dist/types';
-import { SelectChangeEvent } from '@mui/material/Select';
+import {FormikHelpers} from 'formik/dist/types';
+import {SelectChangeEvent} from '@mui/material/Select';
 import CsvMapperRepresentationForm, {
   RepresentationFormEntityOption,
 } from '@components/data/csvMapper/representations/CsvMapperRepresentationForm';
-import { CsvMapper } from '@components/data/csvMapper/CsvMapper';
-import { Theme } from '../../../../components/Theme';
-import { useFormatter } from '../../../../components/i18n';
+import {CsvMapper} from '@components/data/csvMapper/CsvMapper';
+import TextField from 'src/components/TextField';
+import classNames from 'classnames';
+import {Theme} from '../../../../components/Theme';
+import {useFormatter} from '../../../../components/i18n';
 import SwitchField from '../../../../components/SwitchField';
 import useAuth from '../../../../utils/hooks/useAuth';
-import { representationInitialization } from './representations/RepresentationUtils';
+import {representationInitialization} from './representations/RepresentationUtils';
 import CsvMapperTestDialog from './CsvMapperTestDialog';
-import { Representation } from './representations/Representation';
+import {Representation} from './representations/Representation';
 
 const useStyles = makeStyles<Theme>((theme) => ({
   buttons: {
@@ -34,12 +35,23 @@ const useStyles = makeStyles<Theme>((theme) => ({
     display: 'flex',
     alignItems: 'center',
   },
+  marginTop: {
+    marginTop: 20,
+  },
+  formContainer: {
+    margin: '20px 0',
+  },
+  representationContainer: {
+    marginTop: 20,
+    display: 'flex',
+  },
 }));
 
 const csvMapperValidation = (t: (s: string) => string) => Yup.object().shape({
   name: Yup.string().required(t('This field is required')),
   has_header: Yup.boolean().required(t('This field is required')),
   separator: Yup.string().required(t('This field is required')),
+  skipLineChar: Yup.string().max(1),
 });
 
 interface CsvMapperFormProps {
@@ -51,29 +63,29 @@ interface CsvMapperFormProps {
 }
 
 const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
-  csvMapper,
-  onSubmit,
-}) => {
-  const { t } = useFormatter();
+                                                                csvMapper,
+                                                                onSubmit,
+                                                              }) => {
+  const {t} = useFormatter();
   const classes = useStyles();
-
+  
   // -- INIT --
-
+  
   // accordion state
   const [open, setOpen] = useState(false);
-
+  
   // extracting available entities and relationships types from schema
-  const { schema } = useAuth();
+  const {schema} = useAuth();
   const [availableEntityTypes, setAvailableEntityTypes] = useState<
-  RepresentationFormEntityOption[]
+    RepresentationFormEntityOption[]
   >([]);
   const [availableRelationshipTypes, setAvailableRelationshipTypes] = useState<
-  RepresentationFormEntityOption[]
+    RepresentationFormEntityOption[]
   >([]);
-
+  
   // load the available types once in state
   useEffect(() => {
-    const { sdos, scos, smos, sros } = schema;
+    const {sdos, scos, smos, sros} = schema;
     const entityTypes = sdos
       .map((sdo) => ({
         ...sdo,
@@ -99,13 +111,13 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
       value: sro.id,
       type: 'entity_Stix-Core-Relationship',
     }));
-
+    
     setAvailableEntityTypes(entityTypes);
     setAvailableRelationshipTypes(relationshipTypes);
   }, [schema]);
-
+  
   // -- EVENTS --
-
+  
   const onAddEntityRepresentation = (
     setFieldValue: (field: string, value: Representation[]) => void,
     values: CsvMapper,
@@ -131,15 +143,15 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
       representationInitialization('relationship'),
     ]);
   };
-
+  
   // -- ERRORS --
   const [hasError, setHasError] = useState<boolean>(false);
   let errors: Map<string, string> = new Map();
   const handleRepresentationErrors = (key: string, value: boolean) => {
-    errors = { ...errors, [key]: value };
+    errors = {...errors, [key]: value};
     setHasError(Object.values(errors).filter((v) => v).length > 0);
   };
-
+  
   return (
     <>
       <Formik<CsvMapper>
@@ -148,24 +160,24 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
         validationSchema={csvMapperValidation(t)}
         onSubmit={onSubmit}
       >
-        {({ submitForm, isSubmitting, setFieldValue, values }) => {
+        {({submitForm, isSubmitting, setFieldValue, values}) => {
           const entities = values.representations.filter(
             (r) => r.type === 'entity',
           );
           const relationships = values.representations.filter(
             (r) => r.type === 'relationship',
           );
-
+          
           return (
-            <Form style={{ margin: '20px 0 20px 0' }}>
+            <Form className={classes.formContainer}>
               <Field
                 component={TextField}
                 variant="standard"
                 name="name"
                 label={t('Name')}
                 fullWidth
               />
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={classNames(classes.center, classes.marginTop)}>
                 <Field
                   component={SwitchField}
                   type="checkbox"
@@ -180,11 +192,11 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   <InformationOutline
                     fontSize="small"
                     color="primary"
-                    style={{ cursor: 'default' }}
+                    style={{cursor: 'default'}}
                   />
                 </Tooltip>
               </div>
-              <div style={{ marginTop: 20 }}>
+              <div className={classes.marginTop}>
                 <Typography>{t('CSV separator')}</Typography>
                 <div className={classes.center}>
                   <Field
@@ -213,11 +225,30 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   <Typography>{t('Semicolon')}</Typography>
                 </div>
               </div>
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={classes.center}>
+                <Field
+                  component={TextField}
+                  name="skipLineChar"
+                  value={values.skipLineChar}
+                  label={t('Char to escape line')}
+                  onChange={(event: SelectChangeEvent) => setFieldValue('skipLineChar', event.target.value)}
+                />
+                <Tooltip
+                  title={t(
+                    'Every line that begins with this character will be skipped during parsing (for example: #).',
+                  )}
+                >
+                  <InformationOutline
+                    fontSize="small"
+                    color="primary"
+                    style={{cursor: 'default'}}
+                  />
+                </Tooltip>
+              </div>
+              <div className={classNames(classes.center, classes.marginTop)}>
                 <Typography
                   variant="h3"
                   gutterBottom
-                  style={{ marginBottom: 0 }}
                 >
                   {t('Representations for entity')}
                 </Typography>
@@ -228,13 +259,13 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   }
                   size="large"
                 >
-                  <Add fontSize="small" />
+                  <Add fontSize="small"/>
                 </IconButton>
               </div>
               {entities.map((representation, idx) => (
                 <div
                   key={`entity-${idx}`}
-                  style={{ marginTop: 20, display: 'flex' }}
+                  className={classes.representationContainer}
                 >
                   <CsvMapperRepresentationForm
                     key={representation.id}
@@ -245,11 +276,10 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   />
                 </div>
               ))}
-              <div className={classes.center} style={{ marginTop: 20 }}>
+              <div className={classNames(classes.center, classes.marginTop)}>
                 <Typography
                   variant="h3"
                   gutterBottom
-                  style={{ marginBottom: 0 }}
                 >
                   {t('Representations for relationship')}
                 </Typography>
@@ -260,13 +290,13 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   }
                   size="large"
                 >
-                  <Add fontSize="small" />
+                  <Add fontSize="small"/>
                 </IconButton>
               </div>
               {relationships.map((representation, idx) => (
                 <div
                   key={`relationship-${idx}`}
-                  style={{ marginTop: 20, display: 'flex' }}
+                  className={classes.representationContainer}
                 >
                   <CsvMapperRepresentationForm
                     key={representation.id}
@@ -282,7 +312,7 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   variant="contained"
                   color="primary"
                   onClick={() => setOpen(true)}
-                  classes={{ root: classes.button }}
+                  classes={{root: classes.button}}
                   disabled={hasError}
                 >
                   {t('Test')}
@@ -292,7 +322,7 @@ const CsvMapperForm: FunctionComponent<CsvMapperFormProps> = ({
                   color="secondary"
                   onClick={submitForm}
                   disabled={isSubmitting}
-                  classes={{ root: classes.button }}
+                  classes={{root: classes.button}}
                 >
                   {csvMapper.id ? t('Update') : t('Create')}
                 </Button>

opencti-platform/opencti-front/src/schema/relay.schema.graphql

@@ -12403,6 +12403,7 @@ type CsvMapper implements InternalObject & BasicObject {
   name: String!
   has_header: Boolean!
   separator: String!
+  skipLineChar: String
   representations: [CsvMapperRepresentation!]!
   errors: String
 }
@@ -12491,4 +12492,5 @@ input CsvMapperAddInput {
   has_header: Boolean!
   separator: String!
   representations: String!
+  skipLineChar: String
 }

opencti-platform/opencti-front/src/utils/Localization.js

@@ -2210,6 +2210,7 @@ const i18n = {
       'Grantable groups by organization administrators': 'Grupos autorizables por los administradores de la organización',
       'This Group allows the user to bypass restriction. It should not be added here.': 'Este grupo permite al usuario saltarse las restricciones. No debe añadirse aquí',
       'Add a group': 'Añadir un grupo',
+      'Char to escape line': 'Carácter para escapar la línea',
       Processing: 'Procesamiento',
       Automation: 'Automatización',
       'My CSV file contains headers': 'Mi archivo CSV contiene encabezados',
@@ -4432,6 +4433,7 @@ const i18n = {
       'Grantable groups by organization administrators': 'Groupes autorisés par les administrateurs d\'organisations',
       'This Group allows the user to bypass restriction. It should not be added here.': 'Ce groupe permet à l\'utilisateur de contourner les restrictions. Il ne doit pas être ajouté ici',
       'Add a group': 'Ajouter un groupe',
+      'Char to escape line': 'Caractère pour sauter une ligne',
       Processing: 'Traitement',
       Automation: 'Automatisation',
       'My CSV file contains headers': 'Mon fichier CSV contient des en-têtes',
@@ -6573,6 +6575,7 @@ const i18n = {
       'Grantable groups by organization administrators': '組織管理者が付与可能なグループ',
       'This Group allows the user to bypass restriction. It should not be added here.': 'このグループは、ユーザーが制限をバイパスすることができます。ここには追加しないでください、',
       'Add a group': 'グループを追加します',
+      'Char to escape line': 'ラインをエスケープするための文字',
       Processing: '処理',
       Automation: '自動化',
       'My CSV file contains headers': 'CSVファイルにヘッダーが含まれています',
@@ -8610,6 +8613,7 @@ const i18n = {
       'Grantable groups by organization administrators': '组织管理员可授予的组',
       'This Group allows the user to bypass restriction. It should not be added here.': '该组允许用户绕过限制。不应在此添加',
       'Add a group': '添加组',
+      'Char to escape line': 'ラインをエスケープするための文字',
       Processing: '处理',
       Automation: '自动化',
       'My CSV file contains headers': '我的CSV文件包含标题',