Testing: still not stable

README.paths

@@ -1,3 +1,4 @@
+
 Check results are usually written to a path provided by
 option QC_RESULTS in a config file (or on the command-line).
 The default path is identical to the work-space.
@@ -7,11 +8,18 @@ The workspace contains updated and/or user-modified tables,
 a CF test suite (after the first run of dkrz-cf-checker),
 and results of an example run (which is executed
 on the command-line by 'qa-dkrz --example').
-The work-space is $HOME/.qa-dkrz by default.
+Except the unlikely case described below, it is sufficient to 
+apply --workspace=str only once for all.
+The default for a work-space depends on how QA-DKRZ was fetched:
+   conda:  $HOME/.qa-dkrz by default. 
+   GitHub: path to the QA_DKRZ package
 
 $HOME/.qa-dkrz is the home of a 'config' file used to store paths etc. .
 If this doesn't exist, then a file '.conf' is written to QA-DKRZ, i.e
-the package downloaded by 'conda' or from GitHub. If QA-DKRZ is
-without write-permissions, then the user is asked to use --workspace.
-This unlikely case requires --workspace to be provided for each run
-of the script 'QA-DKRZ/scripts/qa-dkrz'.
+the package downloaded by 'conda' or from GitHub. If additionally
+QA-DKRZ is without write-permissions, then the user is asked to 
+use --workspace. This unlikely case requires --workspace to be provided 
+for each run of the script 'QA-DKRZ/scripts/qa-dkrz'.
+
+Please, note for multi-user purpose that $HOME of each users has to exist.
+

example/templates/qa-test.task

@@ -7,7 +7,9 @@
 # Path to QC's results and log dir. Required. This will be expanded
 # automatically by the last path component of option DATA_ROOT.
 # Note: the former QC_DATA_ROOT was replaced by this, but is still valid.
-  QC_RESULTS=test_I
+  QA_RESULTS=results
+
+  PROJECT=CORDEX
 
 # Selection of paths and variables (RegExp of the 'expr' command,
 # i.e. full specification from the beginning of the word).
@@ -22,9 +24,7 @@
 # SELECT := .*/v?[[:digit:]]{3}=  # example for all DRS versions
 
 #  SELECT output/MPI-M/MPI-ESM-LR/L1A85/mon/atmos=
-  SELECT AFR-44/CLMcom/MPI-ESM-LR/historical/.*/CCLM4-8-17=
-
-#  LOCK  AFR-44/CLMcom/MPI-ESM-LR/historical/r1i1p1/CCLM4-8-17=not_
+  SELECT AFR-44/Inst/forcing/historical/.*/model=
 
 # Send a summary of notifications and checked periods of variables
 # to this (list of) email recipients after each finished QC session.
@@ -42,5 +42,5 @@
 # CHECK_MODE=meta  #[data]
 
 # embedded path to another qc configuration file with lower priority
-  QC_CONF=CORDEX_qa.conf
+  QA_CONF=CORDEX_qa.conf
 

include/annotation.h

@@ -54,6 +54,7 @@ class Annotation : public IObj
     bool         entry(void){return false;}
     bool         init(void) ;
     void         linkObject(IObj *){;}
+    void         setFilename(hdhC::FileSplit& f) {file=f;}
     void         setFilename(std::string f){file.setFile(f);}
     void         setProject(std::string s){project=s;}
     void         setTablePath(std::string p){ tablePath=p; }

include/cf.h

@@ -37,8 +37,9 @@ class CF : public IObj
   bool   init(void) ;
   void   linkObject(IObj *);
 
-  void   setFilename(std::string f){file.setFile(f);}
-  void   setTablePath(std::string p){tablePath=p;}
+  void   setFilename(hdhC::FileSplit& f) {file=f;}
+  void   setFilename(std::string f) {file.setFile(f);}
+  void   setTablePath(std::string p) {tablePath=p;}
 
   void   chap(void);
   void   chap_reco(void);

include/date.h

@@ -221,16 +221,17 @@ class Date
       val is considered a Julian day. The bool set true states
       that val is a Julian day anyway.*/
   Date  // val must be a Julian Date number if isRel.==false
-         getDate(double val, bool=false) ;
+         getDate(double val, bool isJD=false) ;
 
   //! Get relative or absolute date.
   /*! Provide time units relative to date and return ISO-8601 string.
       If 'enableSetDate=true', then a reference date is extracted and
       set (see setDate(string) from the std::string parameter before
       a relative part is considered. In case of failure,
-      'not-a-valid-date' string is returned.*/
+      'not-a-valid-date' string is returned.
+      Bool isFormatted=true for formatted value like 20010102.5*/
   Date
-         getDate(std::string, bool enableSetDate=false);
+         getDate(std::string, bool isFormatted=false);
 
   //! Get the day of the year and month.
   double getDay( void );
@@ -343,8 +344,13 @@ class Date
   //! Set date from a Julian date object.
   bool   setDate( const Date::Julian & );
 
-  //! Set date by date-encoded float
-  /*! At present: (%Y%m%d.%f)*/
+  //! Set date by date-encoded float or a Julian Day
+  /*! Encoding:
+        at present: (%Y%m%d.%f); requires previous
+        setting of setFormattedDate().
+      For no encoding:
+        if a date was previously set, then add f to the date,
+        else: f is taken into account a Julian Date.*/
   bool   setDate( double f, std::string cal="", std::string monLen="" );
 
   //! Set reference date.
@@ -389,9 +395,11 @@ class Date
   std::string
          str(void) ;
 
+  void   clear(void);
+
 private:
   enum Calendar { GREGORIAN, PROLEPTIC_GREGORIAN, JULIAN,
-                  EQUAL_MONTHS, ALL_LEAP, NO_LEAP, NONE } ;
+                  EQUAL_MONTHS, ALL_LEAP, NO_LEAP, UNDEF } ;
   enum Calendar currCalendarEnum ;
   std::string currCalendar;
 

include/hdhC.h

@@ -117,6 +117,7 @@ struct FileSplit
    bool isExisting(std::string f="");
    void setExtension(std::string);
    void setFile(std::string f );
+   void setFile(struct FileSplit& );
 
    void setFilename(std::string);
    void setPath(std::string f){ path=f; }

include/in_file.h

@@ -118,8 +118,8 @@ class InFile : public Base
   void   setRecEnd(size_t rec) {ncRecEnd=rec;}
 
   //! Store the path to the netcdf file.
-  void   setFilename(std::string f){file.setFile(f);}
-  void   setTablePath(std::string p){ ; }
+  void   setFilename(std::string f) {file.setFile(f);}
+  void   setTablePath(std::string p) { ; }
 
   struct hdhC::FileSplit file;
 

include/matrix_array.h

@@ -883,35 +883,7 @@ MArep<T>::getDim(std::vector<size_t> &d_in)
       return  ;
 
     // Test dimensions.
-    dim.clear();
-
-    // test, whether any dim size > 0
-    bool is0=true;
-    for(size_t i=0 ; i < d_in.size() ; ++i)
-    {
-      if( d_in[i] )
-        is0=false ;
-    }
-    if( is0 )
-      d_in[0]=1;
-
-    // test, whether any dim size > 1
-    bool isNGT1=true;
-    for(size_t i=0 ; i < d_in.size() ; ++i)
-      if( d_in[i] > 1 )
-        isNGT1=false;
-
-    if( isNGT1 )
-    { // this is in fact not a field, but a point
-      dim.push_back(1);
-      return;
-    }
-
-    // Dimension size(s) of 1 is/are discarded, if
-    // any other is > 1.
-    for(size_t i=0 ; i < d_in.size() ; ++i)
-      if( d_in[i] > 1 )
-        dim.push_back(d_in[i]);
+    dim = d_in;
 
     // Product of dimensions. Purpose: conversion of
     // array index to matrix indices and vice versa.
@@ -999,11 +971,7 @@ void
 MArep<T>::setM(void)
 {
     if( m2D && m2D_dim0 != dim[0] )
-    {
-      delete [] m2D;
-      m2D=0;
-      m2D_dim0=0;
-    }
+       clearM();
 
     // allocate new mem for the 2D shape
     if( dim.size() == 2 )
@@ -1191,7 +1159,7 @@ class MtrxArr : public MtrxArrB
   std::vector<size_t>
           getDimensions(void)const { return rep->dim;}
   size_t  getDimSize(){ return rep->dim.size(); }
-  
+
   size_t  getExceptionCount(size_t i);
   size_t  getExceptionCount(std::string s="all");
 

include/nc_api.h

@@ -166,7 +166,7 @@ class NcAPI
       std::vector<size_t>  rec_leg_begin;
       std::vector<size_t>  rec_leg_sz;
       std::vector<size_t>  rec_leg_end;
-      std::vector<int>     rec_prev;
+      std::vector<void*>   rec_prev_taker;
 
 //! NC4: Compression
       std::vector<int> varShuffle;
@@ -425,12 +425,26 @@ void
 /*! The user has to take care that type conversion is possible.
     Return value: index of x corresponding to rec; -1 for invalid rec */
   template <typename ToT>
-    int
+    ToT
       getData(MtrxArr<ToT> &x, std::string vName, size_t rec=0 );
 
+//! Access of a record by a MtrxArr object.
+/*! Note that data is sliced according to an internally specified maximum
+    range.*/
+    template <typename ToT>
+    ToT
+      getData(MtrxArr<ToT> &x, int varid, size_t rec=0);
+
     void
       getData(std::vector<std::string> &,std::string, size_t rec=0);
 
+    std::pair<int, int>
+      getDataIndexRange(std::string vName)
+        {return getDataIndexRange(getVarID(vName));}
+
+    std::pair<int, int>
+      getDataIndexRange(int varid);
+
 //! Get number of values per record of variable 'vName'.
 /*! If vName is not a valid variable, 0 is returned.
     Do not confuse with number of records.*/
@@ -513,6 +527,14 @@ void
     size_t
       getNumOfRecords(std::string var, bool force=false);
 
+    size_t
+      getRecLegIndex(std::string vName, size_t rec)
+      { return getRecLegIndex(getVarID(vName), rec); }
+
+    size_t
+      getRecLegIndex(int varid, size_t rec)
+        { return rec - layout.rec_leg_begin[varid] ; }
+
 //! Get number of values per record of variable 'vName'.
 /*! If vName is not a valid variable, 0 is returned.
     Do not confuse with number of records.*/
@@ -1035,14 +1057,6 @@ void
     void*
       getData(int varid, size_t rec, size_t leg=0);
 
-//! Access of a record by a MtrxArr object.
-/*! A slice at the rec-th index of the unlimited dimension.
-    If the variable holds 1-D data and leg > rec, then get the leg.
-    Return value: index of x corresponding to rec; -1 for invalid rec  */
-    template <typename ToT>
-    int
-      getData(MtrxArr<ToT> &x, int varid, size_t rec=0);
-
 // Get number of values per record of variable 'vName'.
     size_t
       getDataSize(int varid);

include/qa_CORDEX.h

@@ -103,7 +103,7 @@ class QA : public IObj
   bool   init(void) ;
   void   linkObject(IObj *);
 
-  // special: frim InFile with path and period stripped.
+  // special: from InFile with path and period stripped.
   void   setFilename(hdhC::FileSplit&);
   void   setTablePath(std::string p){tablePath=p;}
 
@@ -317,7 +317,7 @@ class QA : public IObj
   //! Check the path to the tables;
   void   inqTables(void);
 
-  bool   isProgress(void){ return ! isNoProgress ; }
+  bool   isProgress(void){ return ! qaTime.isNoProgress ; }
 
   //! Get coordinates of grid-cells where an error occurred
   /*! Does not work for tripolar coordinates */
@@ -350,6 +350,8 @@ class QA : public IObj
            std::vector<std::string> &,
            std::vector<std::vector<std::string> > &);
 
+  void   resumeSession(void);
+
   //! Connect this with the object to be checked
   void   setInFilePointer(InFile *p) { pIn = p; }
 
@@ -386,7 +388,7 @@ class QA : public IObj
   //! Store results in the internal buffer
   /*! The buffer is flushed to file every 'flushCountMax' time steps.*/
   void   store(std::vector<hdhC::FieldData> &fA);
-  void   storeData(std::vector<hdhC::FieldData> &fA);
+  void   storeData(VariableMetaData&, hdhC::FieldData& );
   void   storeTime(void);
 
   //! Test the time-period of the input file.
@@ -433,7 +435,6 @@ class QA : public IObj
   bool isClearBits;
   bool isFileComplete;
   bool isFirstFile;
-  bool isNoProgress;
   bool isNotFirstRecord;
   bool isPrintTimeBoundDates;
   bool isResumeSession;
@@ -493,7 +494,7 @@ class QA : public IObj
   std::string getCaptIntroDim(VariableMetaData &vMD,
                    struct DimensionMetaData &nc_entry,
                    struct DimensionMetaData &tbl_entry, std::string att="");
-  void        appendToHistory(size_t);
+  void        appendToHistory();
   bool        getExit(void);
   std::string getSubjectsIntroDim(VariableMetaData &vMD,
                    struct DimensionMetaData &nc_entry,

include/qa_time.h

@@ -41,7 +41,7 @@ class TimeOutputBuffer
 
 enum TimeTableMode
 {
-    NONE, REGULAR, ORPHAN, DISABLE, CYCLE
+    UNDEF, REGULAR, ORPHAN, DISABLE, CYCLE
 };
 TimeTableMode timeTableMode;
 
@@ -61,10 +61,10 @@ class QA_Time
   bool   entry(void);
   bool   init(void);
 
-  bool isCheckTime;
   bool isFormattedDate;
   bool isMaxDateRange;
   bool isNoCalendar;
+  bool isNoProgress;
   bool isRegularTimeSteps;
   bool isPrintTimeBoundDates;
   bool isReferenceDate;
@@ -76,17 +76,19 @@ class QA_Time
   std::string boundsName;
   int time_ix;    // the var-index
   int timeBounds_ix;
+  bool isTime;
 
-  double      lastTimeStep;  // is set in flush()
-  double      refTimeOffset;
+  double lastTimeStep;  // is set in flush()
+  double refTimeOffset;
+  double refTimeStep;
 
   Date refDate;
 
   double prevTimeValue ;
   double firstTimeValue;
   double currTimeValue;
   double lastTimeValue;
-  double referenceTimeStep, currTimeStep ;
+  double currTimeStep ;
 
   double prevTimeBoundsValue[2];
   double firstTimeBoundsValue[2];
@@ -118,13 +120,11 @@ class QA_Time
   void   finally(NcAPI *);
 
   void   getDate(Date& , double t);
-  std::string
-         getDateStr(double val, bool isAbsolute=false);
   void   getDRSformattedDateRange(std::vector<Date> &,
                    std::vector<std::string> &);
   void   getTimeBoundsValues(double *pair, size_t rec, double offset=0.);
 
-  void   init(InFile*, Annotation*, QA*);
+  bool   init(InFile*, Annotation*, QA*);
 
   /*! Absolute date encoded by a format given in time:units */
   bool   initAbsoluteTime(std::string &units);