Skip to content

Commit

Permalink
Replicated records: detection and annotation
Browse files Browse the repository at this point in the history
  • Loading branch information
@h-dh
h-dh committed Sep 16, 2015
1 parent 23afc15 commit 5b15342
Show file tree
Hide file tree
Showing 8 changed files with 365 additions and 414 deletions.
3 changes: 3 additions & 0 deletions include/qa_CMIP5.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,9 @@ class QA : public IObj
size_t importedRecFromPrevQA; // initial num of recs in the write-to-nc-file
MtrxArr<double> tmp_mv;

// the same buf-size for all buffer is required for testing replicated records
size_t bufferSize;

// init for test about times
bool enablePostProc;
bool isUseStrict;
Expand Down
3 changes: 3 additions & 0 deletions include/qa_CORDEX.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,9 @@ class QA : public IObj
size_t importedRecFromPrevQA; // initial num of recs in the write-to-nc-file
MtrxArr<double> tmp_mv;

// the same buf-size for all buffer is required for testing replicated records
size_t bufferSize;

// init for test about times
bool enabledPostProc;
bool isUseStrict;
Expand Down
90 changes: 46 additions & 44 deletions include/qa_data.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class DataOutputBuffer
void flush(void);

//! Change the flush counter; 1500 by default
void initBuffer(NcAPI *n, size_t next=0, size_t max=1500);
void initBuffer(QA* , size_t next=0, size_t max=1500);
void setName(std::string n){name=n;}
void setNextFlushBeg(size_t n){nextFlushBeg=n;};
void store(hdhC::FieldData &fA);
Expand All @@ -44,38 +44,9 @@ class DataOutputBuffer
double *ave;
double *std_dev;
int *fill_count;
uint32_t *checksum;
int *checksum;

NcAPI *nc;
};

class SharedRecordFlag
{
public:
SharedRecordFlag();
~SharedRecordFlag();

void adjustFlag(int num, size_t rec, int erase=0);
void flush(void);

//! Change the flush counter; 1500 by default
void initBuffer(NcAPI *, size_t next=0, size_t max=1500);
void setAnnotation(Annotation *p){notes=p;}
void setName(std::string n){name=n;}
void setNextFlushBeg(size_t n){nextFlushBeg=n;};
void store(void);

size_t bufferCount;
size_t maxBufferSize;
size_t nextFlushBeg;

int *buffer;
int currFlag;

std::string name;

Annotation *notes;
NcAPI *nc;
QA* pQA;
};

class Outlier
Expand All @@ -102,9 +73,9 @@ class Outlier

std::vector<std::string> options;

size_t vMDix;
size_t vMD_ix;
std::string name;
MtrxArr<double> tmp_mv;
MtrxArr<double> ma_d;

Annotation *notes;
} ;
Expand All @@ -119,34 +90,64 @@ class ReplicatedRecord
in the array itself.*/

public:
ReplicatedRecord( QA *, size_t vMDix, std::string name);
ReplicatedRecord( QA*, size_t vMDix, std::string name);
~ReplicatedRecord(){;}

void getRange(size_t ix, size_t sz, size_t recNum,
size_t *arr_rep_ix, size_t *arr_1st_ix,
bool *arr_rep_bool, bool *arr_1st_bool,
std::vector<std::string>& range) ;
static bool
isSelected( std::vector<std::string> &options,
std::string &vName,
bool isQA_enablePostProc,
int effDims );
std::string &vName, bool isQA_enablePostProc, int effDims );

void parseOption( std::vector<std::string> &opts ) ;
void report( std::vector<std::string> &note, bool isMultiple );
void report( std::vector<std::string> &note, size_t bufCount,
bool isMultiple );
void setAnnotation(Annotation *p){notes=p;}
void setGroupSize(size_t i){groupSize=i;}
void test( int nRecs, size_t bufferCount, size_t nextFlushBeg,
bool isMultiple, bool isFirst );
bool isMultiple );

bool enableReplicationOnlyGroups;

std::string name;
std::vector<std::string> options;
size_t groupSize;

MtrxArr<double> tmp_mv;
size_t vMD_ix;
Annotation *notes;
QA* pQA;
} ;

class SharedRecordFlag
{
public:
SharedRecordFlag();
~SharedRecordFlag();

void adjustFlag(int num, size_t rec, int erase=0);
void flush(void);

//! Change the flush counter; 1500 by default
void initBuffer(QA*, size_t next=0, size_t max=1500);
void setAnnotation(Annotation *p){notes=p;}
void setName(std::string n){name=n;}
void setNextFlushBeg(size_t n){nextFlushBeg=n;};
void store(void);

size_t bufferCount;
size_t maxBufferSize;
size_t nextFlushBeg;

int *buffer;
int currFlag;

std::string name;

size_t vMDix;
Annotation *notes;
QA *pQA;
} ;
};

class QA_Data
{
Expand Down Expand Up @@ -174,7 +175,7 @@ class QA_Data
void init(InFile*, QA*, std::string);

//! Change the flush counter; 1500 by default
void initBuffer(NcAPI *, size_t next=0, size_t max=1500);
void initBuffer(QA*, size_t next=0, size_t max=1500);

//! Initialisiation of a resumed session.
void initResumeSession(void);
Expand Down Expand Up @@ -225,6 +226,7 @@ class QA_Data
bool isForkedAnnotation;
bool isSingleValueField;

std::string ANNOT_ACCUM;
std::string ANNOT_NO_MT;
size_t numOfClearedBitsInChecksum;

Expand Down
10 changes: 6 additions & 4 deletions include/qa_time.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class TimeOutputBuffer
void flush(void);

//! Change the flush counter; 1500 by default
void initBuffer(NcAPI *n, size_t next=0, size_t max=1500);
void initBuffer(QA*, size_t next=0, size_t max=1500);
void setName(std::string n);
void setNextFlushBeg(size_t n){nextFlushBeg=n;};
void store(double val, double step);
Expand All @@ -32,7 +32,7 @@ class TimeOutputBuffer
size_t maxBufferSize;
size_t nextFlushBeg;

NcAPI *nc;
QA* pQA;

double *buffer;
double *buffer_step;
Expand Down Expand Up @@ -168,8 +168,8 @@ class QA_Time
//! Check time steps
bool testTimeStep(void);

MtrxArr<double> mv_tb;
MtrxArr<double> mv_t;
MtrxArr<double> ma_tb;
MtrxArr<double> ma_t;

// Time Table: hold the state over a record
size_t tt_block_rec ;
Expand All @@ -185,6 +185,8 @@ class QA_Time
size_t maxBufferSize;
size_t nextFlushBeg;

std::string ANNOT_ACCUM;

SharedRecordFlag sharedRecordFlag;
TimeOutputBuffer timeOutputBuffer;

Expand Down
42 changes: 21 additions & 21 deletions src/QA_CMIP5.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ QA::appendToHistory(size_t eCode)
}
else
// not in the history, so take the one from the version attribute
tmp = nc->getAttString("QA-revision");
tmp = nc->getAttString("QA revision");

if( revision != tmp )
{
Expand Down Expand Up @@ -3574,11 +3574,7 @@ QA::init(void)
std::string key("6_15");
if( notes->inq( key, fileStr) )
{
std::string capt("empty data section in the file.") ;

std::string text("Is there an empty data section in the file? Please, check.");
notes->setCheckTimeStr(fail);
notes->setCheckDataStr(fail);
std::string capt("No records in the file.") ;

(void) notes->operate(capt, text) ;
{
Expand Down Expand Up @@ -3614,23 +3610,16 @@ QA::init(void)
}
}

if( isExit )
return true;

// open netCDF for continuation and resuming a session
openQA_Nc(*pIn);

if( isExit || isNoProgress )
if( getExit() || qaTime.isNoProgress )
{
isCheckData=false;
isCheckTime=false;
return true;
}

// This is for the case of processing several layers per record.
// The order must be kept: at first, the time stuff and then
// the field testing.

if( isCheckTime )
{
if( qaTime.isTime && ! pIn->nc.isAnyRecord() )
Expand All @@ -3646,8 +3635,6 @@ QA::init(void)
else
notes->setCheckTimeStr("PASS");
}
else
notes->setCheckTimeStr("OMIT");

if( isCheckData )
{
Expand Down Expand Up @@ -3677,14 +3664,14 @@ QA::initDataOutputBuffer(void)
{
if( isCheckTime )
{
qaTime.timeOutputBuffer.initBuffer(nc, currQARec);
qaTime.sharedRecordFlag.initBuffer(nc, currQARec);
qaTime.timeOutputBuffer.initBuffer(nc, currQARec, bufferSize);
qaTime.sharedRecordFlag.initBuffer(nc, currQARec, bufferSize);
}

if( isCheckData )
{
for( size_t i=0 ; i < varMeDa.size() ; ++i)
varMeDa[i].qaData.initBuffer(nc, currQARec);
varMeDa[i].qaData.initBuffer(nc, currQARec, bufferSize);
}

return;
Expand All @@ -3698,9 +3685,20 @@ QA::initDefaults(void)
nc=0;
notes=0;

cF=0;
pIn=0;
fDI=0;
pOper=0;
pOut=0;
qA=0;
tC=0;

blank=" ";
fail="FAIL";
no_blank="no_blank";
notAvailable="not available";
fileStr="file";
s_colon=":";
s_upper="upper";

enablePostProc=false;
enableVersionInHistory=true;
Expand All @@ -3725,6 +3723,8 @@ QA::initDefaults(void)
isCheckTime=true;
isCheckData=true;

bufferSize=1500;

exitCode=0;

#ifdef REVISION
Expand All @@ -3746,7 +3746,7 @@ QA::initGlobalAtts(InFile &in)
nc->setGlobalAtt( "product", "quality check of CMIP5 data set");

// enableVersionInHistory=false;
nc->setGlobalAtt( "QA_revision", revision);
nc->setGlobalAtt( "QA revision", revision);
nc->setGlobalAtt( "contact", "hollweg@dkrz.de");

std::string t("csv formatted ");
Expand Down

0 comments on commit 5b15342

Please sign in to comment.