Fix reading of csv-files, including caching and tranposing variables for increased performance

git-svn-id: https://openmodelica.org/svn/OpenModelica/trunk@18423 f25d12d1-65f4-0310-ae8a-bbce733d8d8e

Author: sjoelund

Compiler/runtime/SimulationResults.c

@@ -35,7 +35,7 @@ typedef struct {
 #endif
   ModelicaMatReader matReader;
   FILE *pltReader;
-  FILE *csvReader;
+  struct csv_data *csvReader;
 } SimulationResult_Globals;
 
 static SimulationResult_Globals simresglob = {
@@ -48,7 +48,7 @@ static void SimulationResultsImpl__close(SimulationResult_Globals* simresglob)
   switch (simresglob->curFormat) {
   case MATLAB4: omc_free_matlab4_reader(&simresglob->matReader); break;
   case PLT: fclose(simresglob->pltReader); break;
-  case CSV: fclose(simresglob->csvReader); break;
+  case CSV: omc_free_csv_reader(simresglob->csvReader); simresglob->csvReader=NULL; break;
   default: break;
   }
   simresglob->curFormat = UNKNOWN_PLOT;
@@ -103,7 +103,7 @@ static PlotFormat SimulationResultsImpl__openFile(const char *filename, Simulati
     }
     break;
   case CSV:
-    simresglob->csvReader = fopen(filename, "r");
+    simresglob->csvReader = read_csv(filename);
     if (simresglob->csvReader==NULL) {
       msg[1] = filename;
       c_add_message(NULL,-1, ErrorType_scripting, ErrorLevel_error, gettext("Failed to open simulation result %s: %s"), msg, 2);
@@ -119,6 +119,7 @@ static PlotFormat SimulationResultsImpl__openFile(const char *filename, Simulati
   simresglob->curFormat = format;
   simresglob->curFileName = strdup(filename);
 #if !defined(__MINGW32__) && !defined(_MSC_VER)
+  stat(filename, &buf);
   simresglob->mtime = buf.st_mtime;
 #endif
   // fprintf(stderr, "SimulationResultsImpl__openFile(%s) => %s\n", filename, PlotFormatStr[curFormat]);
@@ -216,7 +217,7 @@ static int SimulationResultsImpl__readSimulationResultSize(const char *filename,
     return size;
   }
   case CSV: {
-    size = read_csv_dataset_size(filename);
+    size = simresglob->csvReader ? simresglob->csvReader->numsteps : -1;
     msg[0] = filename;
     if (size == -1) c_add_message(NULL,-1, ErrorType_scripting, ErrorLevel_error, gettext("Failed to read readSimulationResultSize from file: %s\n"), msg, 1);
     return size;
@@ -246,15 +247,14 @@ static void* SimulationResultsImpl__readVars(const char *filename, SimulationRes
     return read_ptolemy_variables(filename);
   }
   case CSV: {
-    int length = 0;
-    char **variables = read_csv_variables(simresglob->csvReader,&length);
-    if (variables) {
+    if (simresglob->csvReader && simresglob->csvReader->variables) {
+      char **variables = simresglob->csvReader->variables;
       int i;
-      for (i=length-1; i>=0; i--) {
-        res = mk_cons(mk_scon(variables[i]),res);
-        GC_free(variables[i]);
+      for (i=simresglob->csvReader->numvars-1; i>=0; i--) {
+        if (variables[i][0] != '\0') {
+          res = mk_cons(mk_scon(variables[i]),res);
+        }
       }
-      GC_free(variables);
     }
     return res;
   }
@@ -349,7 +349,7 @@ static void* SimulationResultsImpl__readDataset(const char *filename, void *vars
     while (RML_NILHDR != RML_GETHDR(vars)) {
       var = RML_STRINGDATA(RML_CAR(vars));
       vars = RML_CDR(vars);
-      vals = read_csv_dataset(filename,var,dimsize);
+      vals = simresglob->csvReader ? read_csv_dataset(simresglob->csvReader,var) : NULL;
       if (vals == NULL) {
         msg[1] = var;
         msg[0] = filename;

SimulationRuntime/c/util/read_csv.c

@@ -81,7 +81,7 @@ static void add_variable(void *data, size_t len, void *t)
     head->buffer_size = head->buffer_size ? 2*head->buffer_size : 512;
     head->variables = GC_realloc(head->variables, sizeof(char*)*head->buffer_size);
   }
-  head->variables[head->size++] = data ? GC_strdup(data) : data;
+  head->variables[head->size++] = GC_strdup(data ? data : "");
 }
 
 static void row_count(int c, void *t)
@@ -149,17 +149,18 @@ char** read_csv_variables(FILE *fin, int *length)
 static void add_cell(void *data, size_t len, void *t)
 {
   struct csv_body *body = (struct csv_body*) t;
-  char *endptr;
+  char *endptr = "";
   if (!body->found_first_row) {
     body->cur_size++;
     body->row_length++;
     return;
   }
   if (body->size+1 >= body->buffer_size) {
-    body->buffer_size = body->buffer_size ? 2*body->buffer_size : body->row_length*1024; /* Guess it's 1024 time points; we could also take the size of the file or something, but this is cool too */
+    body->buffer_size = body->res ? 2*body->buffer_size : body->row_length*1024; /* Guess it's 1024 time points; we could also take the size of the file or something, but this is cool too */
+    body->buffer_size = body->buffer_size > 0 ? body->buffer_size : 1024;
     body->res = body->res ? (double*)GC_realloc(body->res, sizeof(double)*body->buffer_size) : (double*) GC_malloc_atomic(sizeof(double)*body->buffer_size);
   }
-  body->res[body->size++] = strtod((const char*)data,&endptr);
+  body->res[body->size++] = data ? strtod((const char*)data,&endptr) : 0;
   if (*endptr) {
     printf("Found non-double data in csv result-file: %s\n", (char*) data);
     body->error = 1;
@@ -177,7 +178,7 @@ static void add_row(int c, void *t)
   }
 }
 
-double* read_csv_dataset(const char *filename, const char *var, int dimsize)
+double* read_csv_dataset_var(const char *filename, const char *var, int dimsize)
 {
   const int buf_size = 4096;
   char buf[4096];
@@ -203,8 +204,82 @@ double* read_csv_dataset(const char *filename, const char *var, int dimsize)
   csv_fini(&p,add_cell,add_row,&body);
   csv_free(&p);
   fclose(fin);
-  if (!body.error) {
+  if (body.error) {
     return NULL;
   }
   return body.res;
 }
+
+struct csv_data* read_csv(const char *filename)
+{
+  const int buf_size = 4096;
+  char buf[4096];
+  char **variables;
+  int dummy;
+  struct csv_parser p;
+  struct csv_body body = {0};
+  struct csv_data *res = GC_malloc(sizeof(struct csv_data));
+  FILE *fin = fopen(filename, "r");
+  if (!fin) {
+    return NULL;
+  }
+  variables = read_csv_variables(fin,&dummy);
+  if (!variables) {
+    return NULL;
+  }
+  fseek(fin,0,SEEK_SET);
+
+  csv_init(&p, CSV_STRICT | CSV_REPALL_NL | CSV_STRICT_FINI | CSV_APPEND_NULL | CSV_EMPTY_IS_NULL);
+  csv_set_realloc_func(&p, GC_realloc);
+  csv_set_free_func(&p, GC_free);
+  do {
+    size_t len = fread(buf, 1, buf_size, fin);
+    if (len != buf_size && !feof(fin)) {
+      csv_free(&p);
+      fclose(fin);
+      return NULL;
+    }
+    csv_parse(&p,buf,len,add_cell,add_row,&body);
+  } while (!body.error && !feof(fin));
+  csv_fini(&p,add_cell,add_row,&body);
+  csv_free(&p);
+  fclose(fin);
+  if (body.error) {
+    return NULL;
+  }
+  matrix_transpose(body.res,body.row_length,body.size);
+  res->variables = variables;
+  res->data = body.res;
+  res->numvars = body.row_length;
+  res->numsteps = body.size / body.row_length;
+  /* printf("num vars %d in %s num steps %d\n", body.row_length, filename, res->numsteps); */
+  return res;
+}
+
+double* read_csv_dataset(struct csv_data *data, const char *var)
+{
+  int i,found=-1;
+  for (i=0; i<data->numvars; i++) {
+    if (0==strcmp(data->variables[i],var)) {
+      found=i;
+      break;
+    }
+  }
+  if (found == -1) {
+    return NULL;
+  }
+  return data->data + i*data->numsteps;
+}
+
+void omc_free_csv_reader(struct csv_data *data)
+{
+  int i;
+  for (i=0; i<data->numvars; i++) {
+    GC_free(data->variables[i]);
+  }
+  GC_free(data->variables);
+  GC_free(data->data);
+  data->variables = 0;
+  data->data = 0;
+  GC_free(data);
+}

SimulationRuntime/c/util/read_csv.h

@@ -32,10 +32,19 @@
 #ifndef OMC_READ_CSV_H
 #define OMC_READ_CSV_H
 
+struct csv_data {
+  char **variables;
+  double *data;
+  int numvars;
+  int numsteps;
+};
+
 int read_csv_dataset_size(const char* filename);
 
 char** read_csv_variables(FILE *fin, int *length);
 
-double* read_csv_dataset(const char *filename, const char *var, int dimsize);
+struct csv_data* read_csv(const char *filename);
+double* read_csv_dataset(struct csv_data *data, const char *var);
+void omc_free_csv_reader(struct csv_data *data);
 
 #endif

SimulationRuntime/c/util/read_matlab4.c

@@ -447,7 +447,7 @@ double* omc_matlab4_read_vals(ModelicaMatReader *reader, int varIndex)
   return reader->vars[ix];
 }
 
-static void transpose(double *m, int w, int h)
+void matrix_transpose(double *m, int w, int h)
 {
   int start, next, i;
   double tmp;
@@ -495,7 +495,7 @@ int omc_matlab4_read_all_vals(ModelicaMatReader *reader)
       tmp[i] = ((float*)tmp)[i];
     }
   }
-  transpose(tmp,reader->nvar,reader->nrows);
+  matrix_transpose(tmp,reader->nvar,reader->nrows);
   /* Negative aliases */
   for (i=0; i<reader->nrows*reader->nvar; i++) {
     tmp[reader->nrows*reader->nvar + i] = -tmp[i];

SimulationRuntime/c/util/read_matlab4.h

@@ -65,6 +65,8 @@ void omc_matlab4_print_all_vars(FILE *stream, ModelicaMatReader *reader);
 double omc_matlab4_startTime(ModelicaMatReader *reader);
 
 double omc_matlab4_stopTime(ModelicaMatReader *reader);
+void matrix_transpose(double *m, int w, int h);
+
 #ifdef __cplusplus
 } /* extern "C" */
 #endif