Skip to content

Commit

Permalink
Feature 1421 point2grid performance (#1607)
Browse files Browse the repository at this point in the history 
* #1421 Added log message

* #1421 Added log message

* #1421 Performance enhencement for high resolution data. Do not use a cell mapping file

* #1421 read all 2D data at once

* #1421 Added log message

* #1421 Avoid extra extending of IntArray

* #1421 Reset cell mapping for new variable

Co-authored-by: Howard Soh <hsoh@kiowa.rap.ucar.edu>
  • Loading branch information
@hsoh-u
hsoh-u and Howard Soh committed Dec 15, 2020
1 parent 0a18e89 commit 0bfe5bf
Show file tree
Hide file tree
Showing 3 changed files with 234 additions and 295 deletions.
136 changes: 83 additions & 53 deletions met/src/libcode/vx_data2d_nccf/nccf_file.cc
View file
Expand Up @@ -18,6 +18,7 @@ using namespace std;
#include <string.h>
#include <cstdio>
#include <cmath>
#include <ctime>
#include <string>
#include <time.h>

Expand Down Expand Up @@ -754,6 +755,7 @@ int NcCfFile::lead_time() const

double NcCfFile::getData(NcVar * var, const LongArray & a) const
{
clock_t start_clock = clock();
static const string method_name
= "NcCfFile::getData(NcVar *, const LongArray &) -> ";
if (!args_ok(a))
Expand Down Expand Up @@ -863,6 +865,9 @@ double NcCfFile::getData(NcVar * var, const LongArray & a) const

// done

mlog << Debug(6) << method_name << "took "
<< (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n";

return d;
}

Expand All @@ -872,6 +877,7 @@ double NcCfFile::getData(NcVar * var, const LongArray & a) const

bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const
{
clock_t start_clock = clock();
static const string method_name_short
= "NcCfFile::getData(NcVar*, LongArray&, DataPlane&) ";
static const string method_name
Expand Down Expand Up @@ -987,10 +993,11 @@ bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const
}

// get the data
int i[nx];
short s[nx];
float f[nx];
double d[nx];
int *i;
short *s;
double *d;
const int plane_size = nx * ny;
float *f = new float[plane_size];

size_t dim_size;
long offsets[dim_count];
Expand All @@ -1010,6 +1017,9 @@ bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const

offsets[x_slot] = 0;
lengths[x_slot] = nx;
offsets[y_slot] = 0;
lengths[y_slot] = ny;

float add_offset = 0.f;
float scale_factor = 1.f;
NcVarAtt *att_add_offset = get_nc_att(v, (string)"add_offset");
Expand All @@ -1022,67 +1032,87 @@ bool NcCfFile::getData(NcVar * v, const LongArray & a, DataPlane & plane) const
if (att_scale_factor) delete att_scale_factor;

int type_id = GET_NC_TYPE_ID_P(v);
for (int y=0; y<ny; ++y) {
offsets[y_slot] = y;
switch ( type_id ) {

case NcType::nc_SHORT:
get_nc_data(v, (short *)&s, lengths, offsets);
for (int x=0; x<nx; ++x) {
d[x] = (double)s[x];
}
break;
bool do_scale_factor = add_offset != 0.0 || scale_factor != 1.0;

case NcType::nc_INT:
get_nc_data(v, (int *)&i, lengths, offsets);
for (int x=0; x<nx; ++x) {
d[x] = (double)i[x];
}
break;
switch ( type_id ) {

case NcType::nc_FLOAT:
get_nc_data(v, (float *)&f, lengths, offsets);
for (int x=0; x<nx; ++x) {
d[x] = (double)f[x];
}
break;
case NcType::nc_SHORT:
s = new short[plane_size];
get_nc_data(v, s, lengths, offsets);
for (int x=0; x<plane_size; ++x) f[x] = (float)s[x];
delete [] s;
break;

//case ncDouble:
case NcType::nc_DOUBLE:
get_nc_data(v, (double *)&d, lengths, offsets);
break;
case NcType::nc_INT:
i = new int[plane_size];
get_nc_data(v, i, lengths, offsets);
for (int x=0; x<plane_size; ++x) f[x] = (float)i[x];
delete [] i;
break;

default:
mlog << Error << "\n" << method_name
<< " bad type [" << GET_NC_TYPE_NAME_P(v)
<< "] for variable \"" << (GET_NC_NAME_P(v)) << "\"\n\n";
exit ( 1 );
break;
case NcType::nc_FLOAT:
get_nc_data(v, f, lengths, offsets);
break;

} // switch
case NcType::nc_DOUBLE:
d = new double[plane_size];
get_nc_data(v, d, lengths, offsets);
for (int x=0; x<plane_size; ++x) f[x] = (float)d[x];
delete [] d;
break;

default:
mlog << Error << "\n" << method_name
<< " bad type [" << GET_NC_TYPE_NAME_P(v)
<< "] for variable \"" << (GET_NC_NAME_P(v)) << "\"\n\n";
exit ( 1 );
break;

} // switch

LongArray b = a;
y_offset = y;
if (swap_to_north) y_offset = ny - 1 - y;
int offset = 0;
if( x_slot > y_slot ) {
for (int y=0; y<ny; ++y) {
y_offset = y;
if (swap_to_north) y_offset = ny - 1 - y;

for (int x = 0; x< nx; ++x)
{
//double value = getData(v, b);
double value = d[x];
for (int x = 0; x< nx; ++x) {
float value = f[offset++];

if( is_eq(value, missing_value) || is_eq(value, fill_value) ) {
value = bad_data_double;
}
else if( do_scale_factor ) value = value * scale_factor + add_offset;

plane.set(value, x, y_offset);

} // for x
} // for y
}
else {
for (int x = 0; x< nx; ++x) {
for (int y=0; y<ny; ++y) {
y_offset = y;
if (swap_to_north) y_offset = ny - 1 - y;

if(is_eq(value, missing_value) || is_eq(value, fill_value)) {
value = bad_data_double;
}
else if (add_offset != 0.0 || scale_factor != 1.0) {
value = value * scale_factor + add_offset;
}
float value = f[offset++];

plane.set(value, x, y_offset);
if( is_eq(value, missing_value) || is_eq(value, fill_value) ) {
value = bad_data_double;
}
else if( do_scale_factor ) value = value * scale_factor + add_offset;

plane.set(value, x, y_offset);

} // for y
} // for x
}

} // for y
} // for x
delete [] f;

// done
mlog << Debug(6) << method_name << "took "
<< (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n";

return true;
}
Expand Down
12 changes: 10 additions & 2 deletions met/src/libcode/vx_nc_util/nc_utils.cc
View file
Expand Up @@ -1395,6 +1395,7 @@ void _apply_scale_factor(float *data, const T *packed_data,
int unpacked_count = 0;
float min_value = 10e10;
float max_value = -10e10;
clock_t start_clock = clock();
const char *method_name = "apply_scale_factor(float)";

for (int idx=0; idx<cell_count; idx++) {
Expand All @@ -1412,16 +1413,19 @@ void _apply_scale_factor(float *data, const T *packed_data,
}
mlog << Debug(4) << method_name << " unpacked data: count="
<< unpacked_count << " out of " << cell_count
<< ". FillValue(" << data_type << ")=" << fill_value
<< " data range [" << min_value << " - " << max_value
<< ". FillValue(" << data_type << ")=" << fill_value << "\n";
mlog << Debug(4) << method_name << "data range [" << min_value << " - " << max_value
<< "] raw data: [" << raw_min_val << " - " << raw_max_val << "] Positive count: "
<< positive_cnt << "\n";
mlog << Debug(7) << method_name << " took "
<< (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n";
return;
}

////////////////////////////////////////////////////////////////////////

bool get_nc_data(NcVar *var, float *data) {
clock_t start_clock = clock();
bool return_status = false;
static const char *method_name = "get_nc_data(NcVar *, float *) ";

Expand Down Expand Up @@ -1627,6 +1631,10 @@ bool get_nc_data(NcVar *var, float *data) {
}
return_status = true;
}

mlog << Debug(6) << method_name << "took "
<< (clock()-start_clock)/double(CLOCKS_PER_SEC)
<< " seconds for " << GET_NC_NAME_P(var) << "\n";
return(return_status);
}

Expand Down

0 comments on commit 0bfe5bf

Please sign in to comment.