Skip to content

Commit

Permalink
Fix reading of csv-files, including caching and tranposing variables …
Browse files Browse the repository at this point in the history 
…for increased performance

git-svn-id: https://openmodelica.org/svn/OpenModelica/trunk@18423 f25d12d1-65f4-0310-ae8a-bbce733d8d8e
  • Loading branch information
@sjoelund
sjoelund committed Dec 5, 2013
1 parent 6e97464 commit 788c611
Show file tree
Hide file tree
Showing 5 changed files with 107 additions and 21 deletions.
24 changes: 12 additions & 12 deletions Compiler/runtime/SimulationResults.c
View file
Expand Up @@ -35,7 +35,7 @@ typedef struct {
#endif
ModelicaMatReader matReader;
FILE *pltReader;
FILE *csvReader;
struct csv_data *csvReader;
} SimulationResult_Globals;

static SimulationResult_Globals simresglob = {
Expand All @@ -48,7 +48,7 @@ static void SimulationResultsImpl__close(SimulationResult_Globals* simresglob)
switch (simresglob->curFormat) {
case MATLAB4: omc_free_matlab4_reader(&simresglob->matReader); break;
case PLT: fclose(simresglob->pltReader); break;
case CSV: fclose(simresglob->csvReader); break;
case CSV: omc_free_csv_reader(simresglob->csvReader); simresglob->csvReader=NULL; break;
default: break;
}
simresglob->curFormat = UNKNOWN_PLOT;
Expand Down Expand Up @@ -103,7 +103,7 @@ static PlotFormat SimulationResultsImpl__openFile(const char *filename, Simulati
}
break;
case CSV:
simresglob->csvReader = fopen(filename, "r");
simresglob->csvReader = read_csv(filename);
if (simresglob->csvReader==NULL) {
msg[1] = filename;
c_add_message(NULL,-1, ErrorType_scripting, ErrorLevel_error, gettext("Failed to open simulation result %s: %s"), msg, 2);
Expand All @@ -119,6 +119,7 @@ static PlotFormat SimulationResultsImpl__openFile(const char *filename, Simulati
simresglob->curFormat = format;
simresglob->curFileName = strdup(filename);
#if !defined(__MINGW32__) && !defined(_MSC_VER)
stat(filename, &buf);
simresglob->mtime = buf.st_mtime;
#endif
// fprintf(stderr, "SimulationResultsImpl__openFile(%s) => %s\n", filename, PlotFormatStr[curFormat]);
Expand Down Expand Up @@ -216,7 +217,7 @@ static int SimulationResultsImpl__readSimulationResultSize(const char *filename,
return size;
}
case CSV: {
size = read_csv_dataset_size(filename);
size = simresglob->csvReader ? simresglob->csvReader->numsteps : -1;
msg[0] = filename;
if (size == -1) c_add_message(NULL,-1, ErrorType_scripting, ErrorLevel_error, gettext("Failed to read readSimulationResultSize from file: %s\n"), msg, 1);
return size;
Expand Down Expand Up @@ -246,15 +247,14 @@ static void* SimulationResultsImpl__readVars(const char *filename, SimulationRes
return read_ptolemy_variables(filename);
}
case CSV: {
int length = 0;
char **variables = read_csv_variables(simresglob->csvReader,&length);
if (variables) {
if (simresglob->csvReader && simresglob->csvReader->variables) {
char **variables = simresglob->csvReader->variables;
int i;
for (i=length-1; i>=0; i--) {
res = mk_cons(mk_scon(variables[i]),res);
GC_free(variables[i]);
for (i=simresglob->csvReader->numvars-1; i>=0; i--) {
if (variables[i][0] != '\0') {
res = mk_cons(mk_scon(variables[i]),res);
}
}
GC_free(variables);
}
return res;
}
Expand Down Expand Up @@ -349,7 +349,7 @@ static void* SimulationResultsImpl__readDataset(const char *filename, void *vars
while (RML_NILHDR != RML_GETHDR(vars)) {
var = RML_STRINGDATA(RML_CAR(vars));
vars = RML_CDR(vars);
vals = read_csv_dataset(filename,var,dimsize);
vals = simresglob->csvReader ? read_csv_dataset(simresglob->csvReader,var) : NULL;
if (vals == NULL) {
msg[1] = var;
msg[0] = filename;
Expand Down
87 changes: 81 additions & 6 deletions SimulationRuntime/c/util/read_csv.c
View file
Expand Up @@ -81,7 +81,7 @@ static void add_variable(void *data, size_t len, void *t)
head->buffer_size = head->buffer_size ? 2*head->buffer_size : 512;
head->variables = GC_realloc(head->variables, sizeof(char*)*head->buffer_size);
}
head->variables[head->size++] = data ? GC_strdup(data) : data;
head->variables[head->size++] = GC_strdup(data ? data : "");
}

static void row_count(int c, void *t)
Expand Down Expand Up @@ -149,17 +149,18 @@ char** read_csv_variables(FILE *fin, int *length)
static void add_cell(void *data, size_t len, void *t)
{
struct csv_body *body = (struct csv_body*) t;
char *endptr;
char *endptr = "";
if (!body->found_first_row) {
body->cur_size++;
body->row_length++;
return;
}
if (body->size+1 >= body->buffer_size) {
body->buffer_size = body->buffer_size ? 2*body->buffer_size : body->row_length*1024; /* Guess it's 1024 time points; we could also take the size of the file or something, but this is cool too */
body->buffer_size = body->res ? 2*body->buffer_size : body->row_length*1024; /* Guess it's 1024 time points; we could also take the size of the file or something, but this is cool too */
body->buffer_size = body->buffer_size > 0 ? body->buffer_size : 1024;
body->res = body->res ? (double*)GC_realloc(body->res, sizeof(double)*body->buffer_size) : (double*) GC_malloc_atomic(sizeof(double)*body->buffer_size);
}
body->res[body->size++] = strtod((const char*)data,&endptr);
body->res[body->size++] = data ? strtod((const char*)data,&endptr) : 0;
if (*endptr) {
printf("Found non-double data in csv result-file: %s\n", (char*) data);
body->error = 1;
Expand All @@ -177,7 +178,7 @@ static void add_row(int c, void *t)
}
}

double* read_csv_dataset(const char *filename, const char *var, int dimsize)
double* read_csv_dataset_var(const char *filename, const char *var, int dimsize)
{
const int buf_size = 4096;
char buf[4096];
Expand All @@ -203,8 +204,82 @@ double* read_csv_dataset(const char *filename, const char *var, int dimsize)
csv_fini(&p,add_cell,add_row,&body);
csv_free(&p);
fclose(fin);
if (!body.error) {
if (body.error) {
return NULL;
}
return body.res;
}

struct csv_data* read_csv(const char *filename)
{
const int buf_size = 4096;
char buf[4096];
char **variables;
int dummy;
struct csv_parser p;
struct csv_body body = {0};
struct csv_data *res = GC_malloc(sizeof(struct csv_data));
FILE *fin = fopen(filename, "r");
if (!fin) {
return NULL;
}
variables = read_csv_variables(fin,&dummy);
if (!variables) {
return NULL;
}
fseek(fin,0,SEEK_SET);

csv_init(&p, CSV_STRICT | CSV_REPALL_NL | CSV_STRICT_FINI | CSV_APPEND_NULL | CSV_EMPTY_IS_NULL);
csv_set_realloc_func(&p, GC_realloc);
csv_set_free_func(&p, GC_free);
do {
size_t len = fread(buf, 1, buf_size, fin);
if (len != buf_size && !feof(fin)) {
csv_free(&p);
fclose(fin);
return NULL;
}
csv_parse(&p,buf,len,add_cell,add_row,&body);
} while (!body.error && !feof(fin));
csv_fini(&p,add_cell,add_row,&body);
csv_free(&p);
fclose(fin);
if (body.error) {
return NULL;
}
matrix_transpose(body.res,body.row_length,body.size);
res->variables = variables;
res->data = body.res;
res->numvars = body.row_length;
res->numsteps = body.size / body.row_length;
/* printf("num vars %d in %s num steps %d\n", body.row_length, filename, res->numsteps); */
return res;
}

double* read_csv_dataset(struct csv_data *data, const char *var)
{
int i,found=-1;
for (i=0; i<data->numvars; i++) {
if (0==strcmp(data->variables[i],var)) {
found=i;
break;
}
}
if (found == -1) {
return NULL;
}
return data->data + i*data->numsteps;
}

void omc_free_csv_reader(struct csv_data *data)
{
int i;
for (i=0; i<data->numvars; i++) {
GC_free(data->variables[i]);
}
GC_free(data->variables);
GC_free(data->data);
data->variables = 0;
data->data = 0;
GC_free(data);
}
11 changes: 10 additions & 1 deletion SimulationRuntime/c/util/read_csv.h
View file
Expand Up @@ -32,10 +32,19 @@
#ifndef OMC_READ_CSV_H
#define OMC_READ_CSV_H

struct csv_data {
char **variables;
double *data;
int numvars;
int numsteps;
};

int read_csv_dataset_size(const char* filename);

char** read_csv_variables(FILE *fin, int *length);

double* read_csv_dataset(const char *filename, const char *var, int dimsize);
struct csv_data* read_csv(const char *filename);
double* read_csv_dataset(struct csv_data *data, const char *var);
void omc_free_csv_reader(struct csv_data *data);

#endif
4 changes: 2 additions & 2 deletions SimulationRuntime/c/util/read_matlab4.c
View file
Expand Up @@ -447,7 +447,7 @@ double* omc_matlab4_read_vals(ModelicaMatReader *reader, int varIndex)
return reader->vars[ix];
}

static void transpose(double *m, int w, int h)
void matrix_transpose(double *m, int w, int h)
{
int start, next, i;
double tmp;
Expand Down Expand Up @@ -495,7 +495,7 @@ int omc_matlab4_read_all_vals(ModelicaMatReader *reader)
tmp[i] = ((float*)tmp)[i];
}
}
transpose(tmp,reader->nvar,reader->nrows);
matrix_transpose(tmp,reader->nvar,reader->nrows);
/* Negative aliases */
for (i=0; i<reader->nrows*reader->nvar; i++) {
tmp[reader->nrows*reader->nvar + i] = -tmp[i];
Expand Down
2 changes: 2 additions & 0 deletions SimulationRuntime/c/util/read_matlab4.h
View file
Expand Up @@ -65,6 +65,8 @@ void omc_matlab4_print_all_vars(FILE *stream, ModelicaMatReader *reader);
double omc_matlab4_startTime(ModelicaMatReader *reader);

double omc_matlab4_stopTime(ModelicaMatReader *reader);
void matrix_transpose(double *m, int w, int h);

#ifdef __cplusplus
} /* extern "C" */
#endif
Expand Down

0 comments on commit 788c611

Please sign in to comment.