Backport quoted ident (#12349)

* Fixing quoted identifier lexing (#12250) * Fixing quoted identifier lexing - Allow empty quoted ident: '' - Allow quoted ident with double quots: '"' - Sanitize quoted identifiers to fix compilation and linearization * Removing warning in QuotedIdentifier test (#12310) * Fixing quoted identifiers in CSV and FMU (#12346) * Fixing FMU export with quoted identifier * Fixing CSV quoted identifier escape. * Don't allow empty quoted identifier in Modelica: `''`
OpenModelica · Apr 29, 2024 · aa27cc1 · aa27cc1
1 parent 8c01251
commit aa27cc1
Show file tree

Hide file tree

Showing 17 changed files with 459 additions and 49 deletions.
diff --git a/OMCompiler/Compiler/FrontEnd/ModelicaBuiltin.mo b/OMCompiler/Compiler/FrontEnd/ModelicaBuiltin.mo
@@ -2963,7 +2963,7 @@ end plotParametric;
 
 function readSimulationResult "Reads a result file, returning a matrix corresponding to the variables and size given."
   input String filename;
-  input VariableNames variables;
+  input VariableNames variables "e.g. {a.b, a[1].b[3].c}, or a single VariableName";
   input Integer size = 0 "0=read any size... If the size is not the same as the result-file, this function fails";
   output Real result[:,:];
 external "builtin";

diff --git a/OMCompiler/Compiler/NFFrontEnd/NFModelicaBuiltin.mo b/OMCompiler/Compiler/NFFrontEnd/NFModelicaBuiltin.mo
@@ -3216,7 +3216,7 @@ end plotParametric;
 
 function readSimulationResult "Reads a result file, returning a matrix corresponding to the variables and size given."
   input String filename;
-  input VariableNames variables;
+  input VariableNames variables "e.g. {a.b, a[1].b[3].c}, or a single VariableName";
   input Integer size = 0 "0=read any size... If the size is not the same as the result-file, this function fails";
   output Real result[:,:];
 external "builtin";

diff --git a/OMCompiler/Compiler/Template/CodegenC.tpl b/OMCompiler/Compiler/Template/CodegenC.tpl
@@ -5243,9 +5243,9 @@ template functionlinearmodel(ModelInfo modelInfo, String modelNamePrefix) "templ
       <%vectorU%>
       <%vectorY%>
       "\n"
-      <%getVarName(vars.stateVars, "x")%>
-      <%getVarName(vars.inputVars, "u")%>
-      <%getVarName(vars.outputVars, "y")%>
+      <%getVarNameC(vars.stateVars, "x")%>
+      <%getVarNameC(vars.inputVars, "u")%>
+      <%getVarNameC(vars.outputVars, "y")%>
       "equation\n"
       "  der(x) = A * x + B * u;\n"
       "  y = C * x + D * u;\n"
@@ -5274,10 +5274,10 @@ template functionlinearmodel(ModelInfo modelInfo, String modelNamePrefix) "templ
       <%vectorY%>
       <%vectorZ%>
       "\n"
-      <%getVarName(vars.stateVars, "x")%>
-      <%getVarName(vars.inputVars, "u")%>
-      <%getVarName(vars.outputVars, "y")%>
-      <%getVarName(vars.algVars, "z")%>
+      <%getVarNameC(vars.stateVars, "x")%>
+      <%getVarNameC(vars.inputVars, "u")%>
+      <%getVarNameC(vars.outputVars, "y")%>
+      <%getVarNameC(vars.algVars, "z")%>
       "equation\n"
       "  der(x) = A * x + B * u;\n"
       "  y = C * x + D * u;\n"
@@ -5415,17 +5415,18 @@ template functionlinearmodelPython(ModelInfo modelInfo, String modelNamePrefix)
   end match
 end functionlinearmodelPython;
 
-template getVarName(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a varables."
+template getVarNameC(list<SimVar> simVars, String arrayName)
+  "template getVarNameC
+   Generates name for a variable inside a C string."
 ::=
   simVars |> var hasindex arrindex fromindex 1 => (match var
     case SIMVAR(__) then
-      <<"  Real '<%arrayName%>_<%crefStrNoUnderscore(name)%>' = <%arrayName%>[<%arrindex%>];\n">>
+      <<"  Real '<%arrayName%>_<%Util.escapeModelicaStringToCString(escapeSingleQuoteIdent(crefStrNoUnderscore(name)))%>' = <%arrayName%>[<%arrindex%>];\n">>
     end match) ;separator="\n"
-end getVarName;
+end getVarNameC;
 
 template getVarNameMatlab(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a varables."
+  Generates name for a variable."
 ::=
   simVars |> var hasindex arrindex fromindex 1 => (match var
     case SIMVAR(__) then
@@ -5434,7 +5435,7 @@ template getVarNameMatlab(list<SimVar> simVars, String arrayName) "template getV
 end getVarNameMatlab;
 
 template getVarNamePython(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a variables."
+  Generates name for a variable."
 ::=
   simVars |> var hasindex arrindex fromindex 0 => (match var
     case SIMVAR(__) then
@@ -5443,7 +5444,7 @@ template getVarNamePython(list<SimVar> simVars, String arrayName) "template getV
 end getVarNamePython;
 
 template getVarNameJulia(list<SimVar> simVars, String arrayName) "template getVarName
-  Generates name for a varables."
+  Generates name for a variable."
 ::=
   simVars |> var hasindex arrindex fromindex 0 => (match var
     case SIMVAR(__) then

diff --git a/OMCompiler/Compiler/Template/CodegenFMUCommon.tpl b/OMCompiler/Compiler/Template/CodegenFMUCommon.tpl
@@ -475,6 +475,7 @@ template ScalarVariableAttribute2(SimVar simVar, SimCode simCode)
 ::=
 match simVar
   case SIMVAR(__) then
+  let name = Util.escapeModelicaStringToXmlString(System.stringReplace(crefStrNoUnderscore(Util.getOption(exportVar)),"$", "_D_"))
   let defaultValueReference = '<%System.tmpTick()%>'
   let valueReference = getValueReference(simVar, simCode, false)
   let description = if comment then 'description="<%Util.escapeModelicaStringToXmlString(comment)%>"'
@@ -484,7 +485,7 @@ match simVar
   let caus = getCausality2(causality)
   let initial = getFmiInitialAttributeStr(simVar)
   <<
-  name="<%System.stringReplace(crefStrNoUnderscore(Util.getOption(exportVar)),"$", "_D_")%>"
+  name="<%name%>"
   valueReference="<%valueReference%>"
   <%description%>
   <%if boolNot(stringEq(variability_, "")) then 'variability="'+variability_+'"' %>

diff --git a/OMCompiler/Compiler/Template/CodegenUtil.tpl b/OMCompiler/Compiler/Template/CodegenUtil.tpl
@@ -104,11 +104,15 @@ template crefStr(ComponentRef cr)
 end crefStr;
 
 template crefStrNoUnderscore(ComponentRef cr)
- "Generates the name of a variable for variable name array. However does not use underscores on qualified names.
- a.b not a._b. Used for generating variable names that are exported e.g. xml files"
+ "Generates the name of a variable for variable name array.
+  However does not use underscores on qualified names.
+  a.b not a._b.
+
+  Used for generating variable names that are exported e.g. to xml files"
 ::=
   match cr
-  case CREF_IDENT(__) then '<%ident%><%subscriptsStr(subscriptLst)%>'
+  case CREF_IDENT(__) then
+    '<%ident%><%subscriptsStr(subscriptLst)%>'
   case CREF_QUAL(ident = "$DER") then 'der(<%crefStrNoUnderscore(componentRef)%>)'
   case CREF_QUAL(ident = "$CLKPRE") then 'previous(<%crefStrNoUnderscore(componentRef)%>)'
   case CREF_QUAL(__) then '<%ident%><%subscriptsStr(subscriptLst)%>.<%crefStrNoUnderscore(componentRef)%>'
@@ -194,6 +198,13 @@ end crefCCommentWithVariability;
 
 /*********************************************************/
 
+template escapeSingleQuoteIdent(String ident)
+  "Escape single quotes from quoted identifiers."
+::=
+  // TODO: How to replace "'" with "\'" but not "\'" with "\\'"?
+  '<%System.stringReplace(System.stringReplace(ident, "\\'", "\\\\'"), "'", "\\'")%>'
+end escapeSingleQuoteIdent;
+
 template initDefaultValXml(DAE.Type type_)
 ::=
   match type_

diff --git a/OMCompiler/Compiler/Template/CodegenUtilSimulation.tpl b/OMCompiler/Compiler/Template/CodegenUtilSimulation.tpl
@@ -141,21 +141,21 @@ template dumpEqsWork(list<SimEqSystem> eqs)
       <<
       equation index: <%equationIndex(eq)%>
       type: SIMPLE_ASSIGN
-      <%dumpCref(e.cref)%> = <%escapeCComments(dumpExp(e.exp,"\""))%>
+      <%escapeCComments(dumpCref(e.cref))%> = <%escapeCComments(dumpExp(e.exp,"\""))%>
       >>
     case e as SES_SIMPLE_ASSIGN_CONSTRAINTS(__) then
       <<
       equation index: <%equationIndex(eq)%>
       type: SIMPLE_ASSIGN_CONSTRAINTS
-      <%dumpCref(e.cref)%> = <%escapeCComments(dumpExp(e.exp,"\""))%>
+      <%escapeCComments(dumpCref(e.cref))%> = <%escapeCComments(dumpExp(e.exp,"\""))%>
       constraints: <%escapeCComments(dumpConstraints(e.cons))%>
       >>
     case e as SES_ARRAY_CALL_ASSIGN(lhs=lhs as CREF(__)) then
       <<
       equation index: <%equationIndex(eq)%>
       type: ARRAY_CALL_ASSIGN
 
-      <%dumpCref(lhs.componentRef)%> = <%escapeCComments(dumpExp(e.exp,"\""))%>
+      <%escapeCComments(dumpCref(lhs.componentRef))%> = <%escapeCComments(dumpExp(e.exp,"\""))%>
       >>
     case e as SES_GENERIC_ASSIGN(__) then
       <<
@@ -185,7 +185,7 @@ template dumpEqsWork(list<SimEqSystem> eqs)
       equation index: <%equationIndex(eq)%>
       type: LINEAR
 
-      <%ls.vars |> SIMVAR(name=cr) => '<var><%dumpCref(cr)%></var>' ; separator = "\n" %>
+      <%ls.vars |> SIMVAR(name=cr) => '<var><%escapeCComments(dumpCref(cr))%></var>' ; separator = "\n" %>
       <row>
         <%ls.beqs |> exp => '<cell><%escapeCComments(dumpExp(exp,"\""))%></cell>' ; separator = "\n" %><%\n%>
       </row>
@@ -218,7 +218,7 @@ template dumpEqsWork(list<SimEqSystem> eqs)
       indexNonlinear: <%nls.indexNonLinearSystem%>
       type: NONLINEAR
 
-      vars: {<%nls.crefs |> cr => '<%dumpCref(cr)%>' ; separator = ", "%>}
+      vars: {<%nls.crefs |> cr => '<%escapeCComments(dumpCref(cr))%>' ; separator = ", "%>}
       eqns: {<%nls.eqs |> eq => '<%equationIndex(eq)%>' ; separator = ", "%>}
       >>
     case e as SES_MIXED(__) then
@@ -231,7 +231,7 @@ template dumpEqsWork(list<SimEqSystem> eqs)
 
       <mixed>
         <continuous index="<%equationIndex(e.cont)%>" />
-        <%e.discVars |> SIMVAR(name=cr) => '<var><%dumpCref(cr)%></var>' ; separator = ","%>
+        <%e.discVars |> SIMVAR(name=cr) => '<var><%escapeCComments(dumpCref(cr))%></var>' ; separator = ","%>
         <%e.discEqs |> eq => '<discrete index="<%equationIndex(eq)%>" />'%>
       </mixed>
       >>
@@ -251,7 +251,7 @@ template dumpEqsWork(list<SimEqSystem> eqs)
       equation index: <%equationIndex(eq)%>
       type: WHEN
 
-      when {<%conditions |> cond => '<%dumpCref(cond)%>' ; separator=", " %>} then
+      when {<%conditions |> cond => '<%escapeCComments(dumpCref(cond))%>' ; separator=", " %>} then
         <%body%>
       end when;
       >>
@@ -269,7 +269,7 @@ template dumpEqsWork(list<SimEqSystem> eqs)
       let &forstatement = buffer ""
       let &forstatement += 'for ' + escapeCComments(dumpExp(e.iter,"\"")) + ' in ' + escapeCComments(dumpExp(e.startIt,"\""))
       let &forstatement += ' : ' + escapeCComments(dumpExp(e.endIt,"\"")) + ' loop<%\n%>'
-      let &forstatement += '  <%dumpCref(e.cref)%> = <%escapeCComments(dumpExp(e.exp,"\""))%>; '
+      let &forstatement += '  <%escapeCComments(dumpCref(e.cref))%> = <%escapeCComments(dumpExp(e.exp,"\""))%>; '
       let &forstatement += 'end for'
       <<
       equation index: <%equationIndex(e)%>

diff --git a/OMCompiler/Parser/BaseModelica_Lexer.g b/OMCompiler/Parser/BaseModelica_Lexer.g
@@ -386,7 +386,7 @@ fragment
 SCHAR : NL | ~('\r' | '\n' | '\\' | '"');
 
 fragment
-SESCAPE : esc='\\' ('\\' | '"' | '\'' | '?' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' |
+SESCAPE : esc='\\' ('\'' | '"' | '\\' | '?' | 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' |
   {
     char chars[2] = {LA(1),'\0'};
     const char *str = chars;
@@ -431,8 +431,9 @@ QIDENT :
          '\'' (QCHAR | SESCAPE) (QCHAR | SESCAPE)* '\'' ;
 
 fragment
-QCHAR :  (DIGIT | NONDIGIT | '!' | '#' | '$' | '%' | '&' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '>' | '=' | '?' | '@' | '[' | ']' | '^' |
-'{' | '}' | '|' | '~' | ' ');
+QCHAR :  (DIGIT | NONDIGIT | '!' | '#' | '$' | '%' | '&' | '(' | ')'
+          | '*' | '+' | ',' | '-' | '.' | '/' | ':' | ';' | '<' | '>' | '='
+          | '?' | '@' | '[' | ']' | '^' | '{' | '}' | '|' | '~' | ' ' | '"' );
 
 fragment
 NONDIGIT :   ('_' | 'a'..'z' | 'A'..'Z');

diff --git a/OMCompiler/SimulationRuntime/c/simulation/results/simulation_result_csv.cpp b/OMCompiler/SimulationRuntime/c/simulation/results/simulation_result_csv.cpp
@@ -48,8 +48,73 @@
 #include <string.h>
 #include <time.h>
 
+#define MAX_IDENT_LENGTH 4096
+
 extern "C" {
 
+/**
+ * @brief Count the occurrences of a substring in a string
+ *
+ * @param str     String to search.
+ * @param substr  Substring to count in `str`.
+ * @return int    Number of occurrences.
+ */
+int countSubstring(const char* str, const char* substr) {
+    int count = 0;
+    size_t substr_len = strlen(substr);
+    const char* ptr = str;
+    while ((ptr = strstr(ptr, substr)) != NULL) {
+        count++;
+        ptr += substr_len;
+    }
+    return count;
+}
+
+/**
+ * @brief Escape CSV style.
+ *
+ * Escape double-quotes with another double-quote to handle quoted identifiers
+ * with double-quotes in original. Example: `'a"b'` --> `a""b`
+ *
+ * @param original      Variable to escape characters into CSV style.
+ * @param replaced      Buffer to store escaped version of `original`.
+ * @param n             Size of buffer `replaced`.
+ */
+void csvEscapedString(const char* original, char* replaced, size_t n, threadData_t* threadData)
+{
+  size_t original_length = strlen(original);
+
+  // Count the occurrences of \"
+  int num_occurrences = countSubstring(original, "\\\"");
+  size_t replaced_length = original_length + num_occurrences;
+
+  if (replaced == NULL || n < replaced_length + 1) {
+    throwStreamPrint(threadData, "Buffer too small. Failed to escape identifier for CSV result file.");
+    return;
+  }
+
+  size_t j = 0;
+  for (size_t i = 0; i < original_length; i++) {
+      if (original[i] == '"') {
+          replaced[j++] = '"';
+          replaced[j++] = '"';
+      } else {
+          replaced[j++] = original[i];
+      }
+  }
+
+  // Null-terminate the replaced string
+  replaced[j] = '\0';
+  return;
+}
+
+/**
+ * @brief Write CSV data row.
+ *
+ * @param self        Simulation result.
+ * @param data        Simulation data.
+ * @param threadData  Thread data for error handling.
+ */
 void omc_csv_emit(simulation_result *self, DATA *data, threadData_t *threadData)
 {
   FILE *fout = (FILE*) self->storage;
@@ -112,36 +177,63 @@ void omc_csv_emit(simulation_result *self, DATA *data, threadData_t *threadData)
   rt_accumulate(SIM_TIMER_OUTPUT);
 }
 
+/**
+ * @brief Write CSV header.
+ *
+ * @param self        Simulation result.
+ * @param data        Simulation data.
+ * @param threadData  Thread data for error handling.
+ */
 void omc_csv_init(simulation_result *self, DATA *data, threadData_t *threadData)
 {
   int i;
   const MODEL_DATA *mData = data->modelData;
 
   const char* format = ",\"%s\"";
   FILE *fout = omc_fopen(self->filename, "w");
+  char escapedNameBuffer[MAX_IDENT_LENGTH];
 
   assertStreamPrint(threadData, 0!=fout, "Error, couldn't create output file: [%s] because of %s", self->filename, strerror(errno));
 
   fprintf(fout, "\"time\"");
-  if(self->cpuTime)
+  if(self->cpuTime) {
     fprintf(fout, format, "$cpuTime");
-  for(i = 0; i < mData->nVariablesReal; i++) if(!mData->realVarsData[i].filterOutput)
-    fprintf(fout, format, mData->realVarsData[i].info.name);
-  for(i = 0; i < mData->nVariablesInteger; i++) if(!mData->integerVarsData[i].filterOutput)
-    fprintf(fout, format, mData->integerVarsData[i].info.name);
-  for(i = 0; i < mData->nVariablesBoolean; i++) if(!mData->booleanVarsData[i].filterOutput)
-    fprintf(fout, format, mData->booleanVarsData[i].info.name);
-  //for(i = 0; i < mData->nVariablesString; i++) if(!mData->stringVarsData[i].filterOutput)
-  //  fprintf(fout, format, mData->stringVarsData[i].info.name);
-
-  for(i = 0; i < mData->nAliasReal; i++) if(!mData->realAlias[i].filterOutput && data->modelData->realAlias[i].aliasType != 1)
-    fprintf(fout, format, mData->realAlias[i].info.name);
-  for(i = 0; i < mData->nAliasInteger; i++) if(!mData->integerAlias[i].filterOutput && data->modelData->integerAlias[i].aliasType != 1)
-    fprintf(fout, format, mData->integerAlias[i].info.name);
-  for(i = 0; i < mData->nAliasBoolean; i++) if(!mData->booleanAlias[i].filterOutput && data->modelData->booleanAlias[i].aliasType != 1)
-    fprintf(fout, format, mData->booleanAlias[i].info.name);
-  //for(i = 0; i < mData->nAliasString; i++) if(!mData->stringAlias[i].filterOutput && data->modelData->stringAlias[i].aliasType != 1)
-  //  fprintf(fout, format, mData->stringAlias[i].info.name);
+  }
+  for(i = 0; i < mData->nVariablesReal; i++) if(!mData->realVarsData[i].filterOutput) {
+    csvEscapedString(mData->realVarsData[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+    fprintf(fout, format, escapedNameBuffer);
+  }
+  for(i = 0; i < mData->nVariablesInteger; i++) {
+    if(!mData->integerVarsData[i].filterOutput) {
+      csvEscapedString(mData->integerVarsData[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+  for(i = 0; i < mData->nVariablesBoolean; i++) {
+    if(!mData->booleanVarsData[i].filterOutput) {
+      csvEscapedString(mData->booleanVarsData[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+
+  for(i = 0; i < mData->nAliasReal; i++) {
+    if(!mData->realAlias[i].filterOutput && data->modelData->realAlias[i].aliasType != 1) {
+      csvEscapedString(mData->realAlias[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+  for(i = 0; i < mData->nAliasInteger; i++) {
+    if(!mData->integerAlias[i].filterOutput && data->modelData->integerAlias[i].aliasType != 1) {
+      csvEscapedString(mData->integerAlias[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
+  for(i = 0; i < mData->nAliasBoolean; i++) {
+    if(!mData->booleanAlias[i].filterOutput && data->modelData->booleanAlias[i].aliasType != 1) {
+      csvEscapedString(mData->booleanAlias[i].info.name, escapedNameBuffer, MAX_IDENT_LENGTH, threadData);
+      fprintf(fout, format, escapedNameBuffer);
+    }
+  }
   fprintf(fout, "\n");
   self->storage = fout;
 }

diff --git a/testsuite/flattening/modelica/mosfiles/.gitignore b/testsuite/flattening/modelica/mosfiles/.gitignore
@@ -0,0 +1 @@
+/QuotedIdentifier_tmp/
diff --git a/testsuite/flattening/modelica/mosfiles/Makefile b/testsuite/flattening/modelica/mosfiles/Makefile
@@ -28,6 +28,8 @@ ModifierVariabilityError.mos \
 NotbasicType.mos NotbasicType2.mos \
 OverloadingFunc.mos \
 QuotedFunction.mos \
+QuotedIdentifier.mos \
+QuotedIdentifierCSV.mos \
 Return.mos \
 StringArrayReturn.mos \
 TestLoadModel.mos \