Fixing quoted identifiers in CSV and FMU (#12346)

* Fixing FMU export with quoted identifier

* Fixing CSV quoted identifier escape.

* Don't allow empty quoted identifier in Modelica: `''`

OMCompiler/Compiler/FrontEnd/ModelicaBuiltin.mo

@@ -2963,7 +2963,7 @@ end plotParametric;
 
 function readSimulationResult "Reads a result file, returning a matrix corresponding to the variables and size given."
   input String filename;
-  input VariableNames variables;
+  input VariableNames variables "e.g. {a.b, a[1].b[3].c}, or a single VariableName";
   input Integer size = 0 "0=read any size... If the size is not the same as the result-file, this function fails";
   output Real result[:,:];
 external "builtin";

OMCompiler/Compiler/NFFrontEnd/NFModelicaBuiltin.mo

@@ -3216,7 +3216,7 @@ end plotParametric;
 
 function readSimulationResult "Reads a result file, returning a matrix corresponding to the variables and size given."
   input String filename;
-  input VariableNames variables;
+  input VariableNames variables "e.g. {a.b, a[1].b[3].c}, or a single VariableName";
   input Integer size = 0 "0=read any size... If the size is not the same as the result-file, this function fails";
   output Real result[:,:];
 external "builtin";

OMCompiler/Compiler/Template/CodegenC.tpl

@@ -5244,9 +5244,9 @@ template functionlinearmodel(ModelInfo modelInfo, String modelNamePrefix) "templ
       <%vectorU%>
       <%vectorY%>
       "\n"
-      <%getVarName(vars.stateVars, "x")%>
-      <%getVarName(vars.inputVars, "u")%>
-      <%getVarName(vars.outputVars, "y")%>
+      <%getVarNameC(vars.stateVars, "x")%>
+      <%getVarNameC(vars.inputVars, "u")%>
+      <%getVarNameC(vars.outputVars, "y")%>
       "equation\n"
       "  der(x) = A * x + B * u;\n"
       "  y = C * x + D * u;\n"
@@ -5275,10 +5275,10 @@ template functionlinearmodel(ModelInfo modelInfo, String modelNamePrefix) "templ
       <%vectorY%>
       <%vectorZ%>
       "\n"
-      <%getVarName(vars.stateVars, "x")%>
-      <%getVarName(vars.inputVars, "u")%>
-      <%getVarName(vars.outputVars, "y")%>
-      <%getVarName(vars.algVars, "z")%>
+      <%getVarNameC(vars.stateVars, "x")%>
+      <%getVarNameC(vars.inputVars, "u")%>
+      <%getVarNameC(vars.outputVars, "y")%>
+      <%getVarNameC(vars.algVars, "z")%>
       "equation\n"
       "  der(x) = A * x + B * u;\n"
       "  y = C * x + D * u;\n"
@@ -5416,17 +5416,18 @@ template functionlinearmodelPython(ModelInfo modelInfo, String modelNamePrefix)
   end match
 end functionlinearmodelPython;
 
-template getVarName(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a varables."
+template getVarNameC(list<SimVar> simVars, String arrayName)
+  "template getVarNameC
+   Generates name for a variable inside a C string."
 ::=
   simVars |> var hasindex arrindex fromindex 1 => (match var
     case SIMVAR(__) then
-      <<"  Real '<%arrayName%>_<%crefStrNoUnderscore(name)%>' = <%arrayName%>[<%arrindex%>];\n">>
+      <<"  Real '<%arrayName%>_<%Util.escapeModelicaStringToCString(escapeSingleQuoteIdent(crefStrNoUnderscore(name)))%>' = <%arrayName%>[<%arrindex%>];\n">>
     end match) ;separator="\n"
-end getVarName;
+end getVarNameC;
 
 template getVarNameMatlab(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a varables."
+  Generates name for a variable."
 ::=
   simVars |> var hasindex arrindex fromindex 1 => (match var
     case SIMVAR(__) then
@@ -5435,7 +5436,7 @@ template getVarNameMatlab(list<SimVar> simVars, String arrayName) "template getV
 end getVarNameMatlab;
 
 template getVarNamePython(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a variables."
+  Generates name for a variable."
 ::=
   simVars |> var hasindex arrindex fromindex 0 => (match var
     case SIMVAR(__) then
@@ -5444,7 +5445,7 @@ template getVarNamePython(list<SimVar> simVars, String arrayName) "template getV
 end getVarNamePython;
 
 template getVarNameJulia(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a varables."
+  Generates name for a variable."
 ::=
   simVars |> var hasindex arrindex fromindex 0 => (match var
     case SIMVAR(__) then

OMCompiler/Compiler/Template/CodegenFMUCommon.tpl

@@ -475,6 +475,7 @@ template ScalarVariableAttribute2(SimVar simVar, SimCode simCode)
 ::=
 match simVar
   case SIMVAR(__) then
+  let name = Util.escapeModelicaStringToXmlString(System.stringReplace(crefStrNoUnderscore(Util.getOption(exportVar)),"$", "_D_"))
   let defaultValueReference = '<%System.tmpTick()%>'
   let valueReference = getValueReference(simVar, simCode, false)
   let description = if comment then 'description="<%Util.escapeModelicaStringToXmlString(comment)%>"'
@@ -484,7 +485,7 @@ match simVar
   let caus = getCausality2(causality)
   let initial = getFmiInitialAttributeStr(simVar)
   <<
-  name="<%System.stringReplace(crefStrNoUnderscore(Util.getOption(exportVar)),"$", "_D_")%>"
+  name="<%name%>"
   valueReference="<%valueReference%>"
   <%description%>
   <%if boolNot(stringEq(variability_, "")) then 'variability="'+variability_+'"' %>

OMCompiler/Compiler/Template/CodegenUtil.tpl

@@ -107,14 +107,12 @@ template crefStrNoUnderscore(ComponentRef cr)
  "Generates the name of a variable for variable name array.
   However does not use underscores on qualified names.
   a.b not a._b.
-  Escaping single and double quotes from quoted identifiers.
 
-  Used for generating variable names that are exported e.g. xml files"
+  Used for generating variable names that are exported e.g. to xml files"
 ::=
   match cr
   case CREF_IDENT(__) then
-    let escapedIdent = Util.escapeModelicaStringToCString(escapeSingleQuoteIdent(ident))
-    '<%escapedIdent%><%subscriptsStr(subscriptLst)%>'
+    '<%ident%><%subscriptsStr(subscriptLst)%>'
   case CREF_QUAL(ident = "$DER") then 'der(<%crefStrNoUnderscore(componentRef)%>)'
   case CREF_QUAL(ident = "$CLKPRE") then 'previous(<%crefStrNoUnderscore(componentRef)%>)'
   case CREF_QUAL(__) then '<%ident%><%subscriptsStr(subscriptLst)%>.<%crefStrNoUnderscore(componentRef)%>'

OMCompiler/Parser/BaseModelica_Lexer.g

@@ -428,7 +428,7 @@ IDENT2 : NONDIGIT (NONDIGIT | DIGIT)* | '$cpuTime';
 
 fragment
 QIDENT :
-         '\'' (QCHAR | SESCAPE)* '\'' ;
+         '\'' (QCHAR | SESCAPE) (QCHAR | SESCAPE)* '\'' ;
 
 fragment
 QCHAR :  (DIGIT | NONDIGIT | '!' | '#' | '$' | '%' | '&' | '(' | ')'

OMCompiler/SimulationRuntime/c/simulation/results/simulation_result_csv.cpp

@@ -48,8 +48,73 @@
 #include <string.h>
 #include <time.h>
 
+#define MAX_IDENT_LENGTH 4096
+
 extern "C" {
 
+/**
+ * @brief Count the occurrences of a substring in a string
+ *
+ * @param str     String to search.
+ * @param substr  Substring to count in `str`.
+ * @return int    Number of occurrences.
+ */
+int countSubstring(const char* str, const char* substr) {
+    int count = 0;
+    size_t substr_len = strlen(substr);
+    const char* ptr = str;
+    while ((ptr = strstr(ptr, substr)) != NULL) {
+        count++;
+        ptr += substr_len;
+    }
+    return count;
+}
+
+/**
+ * @brief Escape CSV style.
+ *
+ * Escape double-quotes with another double-quote to handle quoted identifiers
+ * with double-quotes in original. Example: `'a"b'` --> `a""b`
+ *
+ * @param original      Variable to escape characters into CSV style.
+ * @param replaced      Buffer to store escaped version of `original`.
+ * @param n             Size of buffer `replaced`.
+ */
+void csvEscapedString(const char* original, char* replaced, size_t n, threadData_t* threadData)
+{
+  size_t original_length = strlen(original);
+
+  // Count the occurrences of \"
+  int num_occurrences = countSubstring(original, "\\\"");
+  size_t replaced_length = original_length + num_occurrences;
+
+  if (replaced == NULL || n < replaced_length + 1) {
+    throwStreamPrint(threadData, "Buffer too small. Failed to escape identifier for CSV result file.");
+    return;
+  }
+
+  size_t j = 0;
+  for (size_t i = 0; i < original_length; i++) {
+      if (original[i] == '"') {
+          replaced[j++] = '"';
+          replaced[j++] = '"';
+      } else {
+          replaced[j++] = original[i];
+      }
+  }
+
+  // Null-terminate the replaced string
+  replaced[j] = '\0';
+  return;
+}
+
+/**
+ * @brief Write CSV data row.
+ *
+ * @param self        Simulation result.
+ * @param data        Simulation data.
+ * @param threadData  Thread data for error handling.
+ */
 void omc_csv_emit(simulation_result *self, DATA *data, threadData_t *threadData)
 {
   FILE *fout = (FILE*) self->storage;
@@ -112,36 +177,63 @@ void omc_csv_emit(simulation_result *self, DATA *data, threadData_t *threadData)
   rt_accumulate(SIM_TIMER_OUTPUT);
 }
 
+/**
+ * @brief Write CSV header.
+ *
+ * @param self        Simulation result.
+ * @param data        Simulation data.
+ * @param threadData  Thread data for error handling.
+ */
 void omc_csv_init(simulation_result *self, DATA *data, threadData_t *threadData)
 {
   int i;
   const MODEL_DATA *mData = data->modelData;
 
   const char* format = ",\"%s\"";
   FILE *fout = omc_fopen(self->filename, "w");
+  char escapedNameBuffer[MAX_IDENT_LENGTH];
 
   assertStreamPrint(threadData, 0!=fout, "Error, couldn't create output file: [%s] because of %s", self->filename, strerror(errno));
 
   fprintf(fout, "\"time\"");
-  if(self->cpuTime)
+  if(self->cpuTime) {
     fprintf(fout, format, "$cpuTime");
-  for(i = 0; i < mData->nVariablesReal; i++) if(!mData->realVarsData[i].filterOutput)
-    fprintf(fout, format, mData->realVarsData[i].info.name);
-  for(i = 0; i < mData->nVariablesInteger; i++) if(!mData->integerVarsData[i].filterOutput)
-    fprintf(fout, format, mData->integerVarsData[i].info.name);
-  for(i = 0; i < mData->nVariablesBoolean; i++) if(!mData->booleanVarsData[i].filterOutput)
-    fprintf(fout, format, mData->booleanVarsData[i].info.name);
-  //for(i = 0; i < mData->nVariablesString; i++) if(!mData->stringVarsData[i].filterOutput)
-  //  fprintf(fout, format, mData->stringVarsData[i].info.name);
-
-  for(i = 0; i < mData->nAliasReal; i++) if(!mData->realAlias[i].filterOutput && data->modelData->realAlias[i].aliasType != 1)
-    fprintf(fout, format, mData->realAlias[i].info.name);
-  for(i = 0; i < mData->nAliasInteger; i++) if(!mData->integerAlias[i].filterOutput && data->modelData->integerAlias[i].aliasType != 1)
-    fprintf(fout, format, mData->integerAlias[i].info.name);
-  for(i = 0; i < mData->nAliasBoolean; i++) if(!mData->booleanAlias[i].filterOutput && data->modelData->booleanAlias[i].aliasType != 1)
-    fprintf(fout, format, mData->booleanAlias[i].info.name);
-  //for(i = 0; i < mData->nAliasString; i++) if(!mData->stringAlias[i].filterOutput && data->modelData->stringAlias[i].aliasType != 1)
-  //  fprintf(fout, format, mData->stringAlias[i].info.name);
+  }
+  for(i = 0; i < mData->nVariablesReal; i++) if(!mData->realVarsData[i].filterOutput) {
+    csvEscapedString(mData->realVarsData[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+    fprintf(fout, format, escapedNameBuffer);
+  }
+  for(i = 0; i < mData->nVariablesInteger; i++) {
+    if(!mData->integerVarsData[i].filterOutput) {
+      csvEscapedString(mData->integerVarsData[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+  for(i = 0; i < mData->nVariablesBoolean; i++) {
+    if(!mData->booleanVarsData[i].filterOutput) {
+      csvEscapedString(mData->booleanVarsData[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+
+  for(i = 0; i < mData->nAliasReal; i++) {
+    if(!mData->realAlias[i].filterOutput && data->modelData->realAlias[i].aliasType != 1) {
+      csvEscapedString(mData->realAlias[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+  for(i = 0; i < mData->nAliasInteger; i++) {
+    if(!mData->integerAlias[i].filterOutput && data->modelData->integerAlias[i].aliasType != 1) {
+      csvEscapedString(mData->integerAlias[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+  for(i = 0; i < mData->nAliasBoolean; i++) {
+    if(!mData->booleanAlias[i].filterOutput && data->modelData->booleanAlias[i].aliasType != 1) {
+      csvEscapedString(mData->booleanAlias[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
   fprintf(fout, "\n");
   self->storage = fout;
 }

testsuite/flattening/modelica/mosfiles/Makefile

@@ -29,6 +29,7 @@ NotbasicType.mos NotbasicType2.mos \
 OverloadingFunc.mos \
 QuotedFunction.mos \
 QuotedIdentifier.mos \
+QuotedIdentifierCSV.mos \
 Return.mos \
 StringArrayReturn.mos \
 TestLoadModel.mos \

testsuite/flattening/modelica/mosfiles/QuotedIdentifier.mo

@@ -4,7 +4,6 @@ model QuotedIdentifier
   Real 'e"f g'(start = 1, fixed = true);
   Real 'h\'i //j'(start = 1, fixed = true);
   Real '\\\\\''(start = 1, fixed = true);
-  Real ''(start = 1, fixed = true);
   Real '*/ no code injection'(start = 1, fixed = true);
   Real '\''(start = 1, fixed = true);
   Real /*(y)*/ 'stupid,name'(start = 1, fixed = true);
@@ -14,7 +13,6 @@ equation
   der('e"f g') = 2;
   der('h\'i //j') = -2;
   der('\\\\\'') = 3;
-  der('') = -3;
   der('*/ no code injection') = 4;
   der('\'') = -4;
   der('stupid,name') = 5;

testsuite/flattening/modelica/mosfiles/QuotedIdentifierCSV.mos

@@ -0,0 +1,31 @@
+// name: QuotedIdentifierCSV
+// status: correct
+// teardown_command: rm -rf QuotedIdentifierCSV_tmp
+//
+// Test CSV result file generation of variables containing commas.
+
+loadFile("QuotedIdentifier.mo");getErrorString();
+
+echo(false);
+mkdir("QuotedIdentifierCSV_tmp"); cd("QuotedIdentifierCSV_tmp");
+echo(true);
+
+simulate(QuotedIdentifier, outputFormat="csv"); getErrorString();
+
+readSimulationResultVars("QuotedIdentifier_res.csv"); getErrorString();
+
+// Result:
+// true
+// ""
+// true
+// record SimulationResult
+//     resultFile = "QuotedIdentifier_res.csv",
+//     simulationOptions = "startTime = 0.0, stopTime = 1.0, numberOfIntervals = 500, tolerance = 1e-6, method = 'dassl', fileNamePrefix = 'QuotedIdentifier', options = '', outputFormat = 'csv', variableFilter = '.*', cflags = '', simflags = ''",
+//     messages = "LOG_SUCCESS       | info    | The initialization finished successfully without homotopy method.
+// LOG_SUCCESS       | info    | The simulation finished successfully.
+// "
+// end SimulationResult;
+// ""
+// {"time","'*/ no code injection'","'\\''","'\\\\\\\\\\''","'a\"b'","'c d'","'e\"f g'","'h\\'i //j'","'stupid,name'","der('*/ no code injection')","der('\\'')","der('\\\\\\\\\\'')","der('a\"b')","der('c d')","der('e\"f g')","der('h\\'i //j')","der('stupid,name')"}
+// ""
+// endResult

testsuite/openmodelica/fmi/ModelExchange/2.0/.gitignore

@@ -0,0 +1 @@
+/QuotedIdentifier_tmp/

testsuite/openmodelica/fmi/ModelExchange/2.0/Makefile

@@ -24,6 +24,7 @@ fmi_attributes_20.mos \
 fmi_attributes_21.mos \
 fmi_attributes_22.mos \
 FMUResourceTest.mos \
+QuotedIdentifierExport.mos \
 testBug2764.mos \
 testBug2765.mos \
 testBug3049.mos \

testsuite/openmodelica/fmi/ModelExchange/2.0/QuotedIdentifier.mo

@@ -0,0 +1,21 @@
+model QuotedIdentifier
+  Real 'a"b'(start = 1, fixed = true);
+  Real 'c d'(start = 2, fixed = true);
+  Real 'e"f g'(start = 3, fixed = true);
+  Real 'h\'i //j'(start = 4, fixed = true);
+  Real '\\\\\''(start = 5, fixed = true);
+  Real '<&>'(start = 6, fixed = true);
+  Real '*/ no code injection'(start = 7, fixed = true);
+  Real '\''(start = 8, fixed = true);
+  Real /*(y)*/ 'stupid,name'(start = 9, fixed = true);
+equation
+  der('a"b') = 1;
+  der('c d') = -1;
+  der('e"f g') = 2;
+  der('h\'i //j') = -2;
+  der('\\\\\'') = 3;
+  der('<&>') = -3;
+  der('*/ no code injection') = 4;
+  der('\'') = -4;
+  der('stupid,name') = 5;
+end QuotedIdentifier;