Outlier test I

IS-ENES-Data · Sep 17, 2015 · ea8e462 · ea8e462
1 parent 73c3849
commit ea8e462
Show file tree

Hide file tree

Showing 4 changed files with 65 additions and 162 deletions.
diff --git a/scripts/qa-config b/scripts/qa-config
@@ -1312,6 +1312,9 @@ parseOPTARG()
   test "${key}" = SHOW_OPT && \
     displayComLineOptions 'dumb' && exit 1
 
+  test "${key}" = SHOW_NEXT && \
+    setKWL SHOW_NEXT=${value:-0} && return
+
   if [ "${key}" = NOTE ] ; then
     for(( i=0 ; i < ${#keyWordList[*]} ; ++i )) ; do
       if [ "${key}" = ${keyWordList[i]} ] ; then
@@ -1549,7 +1552,6 @@ setKWL()
 
   # take deprecated SHOW_EXP_NAME into account
   test "${key:0:8}" = SHOW_EXP && key=SHOW_EXP
-  test "${key:0:9}" = POST_PROC && key=POST_PROC
   test "${key}" = HIDDEN_DIRECTORY_DESCENT && key=HIDDEN_DIRECTORIES
 
 #  eval ${key}="${value// /\\ /}"
@@ -2024,9 +2026,6 @@ do
             setKWL PROGRESS_BAR=${OPTARG#*=}  # file to write to
           fi
 
-        elif [ "${UOPTARG:0:4}" = POST ] ; then
-          setKWL POST_PROC=t
-
         elif [ "${UOPTARG:0:9}" = WORKSPACE ] ; then
           installArgs="${installArgs}QA_WORK=${OPTARG#*=},"
           setKWL QA_WORK=${OPTARG##*=}

diff --git a/scripts/qa-dkrz b/scripts/qa-dkrz
@@ -270,11 +270,6 @@ callQaConfig()
     fi
 
     setKWL ${key}
-
-    # check for key-word POST in any assignment
-    if expr match "${items[i]#*=}" 'POST' &> /dev/null ; then
-      isPost=t
-    fi
   done
 
 # invoked by option -E_DEBUG_MANAGER
@@ -1812,7 +1807,7 @@ getNextVariable()
   fi
 
   nextFile="$(echo ${nV_fls[*]} | ${QA_BIN#*:}/syncFiles.x \
-     ${isPost:+--post} $syncOpts -P ${nextPath} )"
+     $syncOpts -P ${nextPath} )"
 
   status=$?
 
@@ -1821,6 +1816,13 @@ getNextVariable()
 
   local nFsSz=${#nextFile[*]}
 
+  if [ ${status} -gt 49 ] ; then
+    status=$((status - 50))
+    isPost=t
+  else
+    isPost=f
+  fi
+
   # up-to-date
   if [ ${status} -eq 1 ] ; then
 #    if [ ${PROGRESS_BAR} ] ; then
@@ -1830,20 +1832,6 @@ getNextVariable()
       progressFileNum=$(( ${progressFileNum:-0} + num ))
 #    fi
 
-    # --post option caused to output the last filename
-    if [ "${qa_fl[0]}" -a ${isPost:-f} = t ] ; then
-      # check global attributes for outlier_test=done
-#      local att
-#      att=$( getNC_att.x ${qa_fl} postponed_operations )
-#      att=${att#*=}
-#      att=${att%% }
-
-#      if [ ! "${att}" -o ${FORCE_POST:-f} = t ] ; then
-        nextFile=( ${nextFile[*]//\\n/ } )
-#        return 1
-#      fi
-    fi
-
     nextFile=
     return 0
   fi
@@ -2696,8 +2684,7 @@ init()
     tput dl 1 > $TTY
   fi
 
-#  syncOptsInit="-s -T"
-  syncOptsInit='-s --only-marked'
+  syncOptsInit='--only-marked'
 
   # progress bar
 #  if [ ${PROGRESS_BAR:-f} = t ] ; then

diff --git a/src/QA_data.cpp b/src/QA_data.cpp
@@ -477,27 +477,18 @@ Outlier::test(QA_Data *pQAD)
   std::vector<size_t> outRec;
   std::vector<double> outVal;
 
-  size_t currRecEnd=pQA->nc->getNumOfRecords();
-//  size_t subTempRecs=pQA->pIn->nc.getNumOfRecords();
-  size_t prevRecEnd ;
-
-//  if( pQA->enabledPostProc )
-    prevRecEnd=0  ;  //forces re-reading of all
-//  else
-//    prevRecEnd = currRecEnd - subTempRecs ;
-
-  size_t currRecNum = currRecEnd - prevRecEnd ;
+  size_t recNum=pQA->nc->getNumOfRecords();
 
   // adjustment of outOptPrcnt: the effective percentage must be related
   // to the total number of data points in a way that 10 outlier must
   // be possible in principle.
   if( outOptPrcnt > 0. )
   {
-    while ( static_cast<double>(currRecNum) * outOptPrcnt < .1 )
+    while ( static_cast<double>(recNum) * outOptPrcnt < .1 )
      outOptPrcnt *= 10. ;
   }
 
-  double extr[currRecNum];
+  double extr[recNum];
 
   MtrxArr<double> ma_d;
   size_t N[100]; // counter for exceeding extreme values
@@ -509,10 +500,11 @@ Outlier::test(QA_Data *pQAD)
 
     bool isOut = false;
     size_t rec=0;
-    for( size_t j=prevRecEnd ; j < currRecEnd ; ++j )
+    for( size_t j=0 ; j < recNum ; ++j )
     {
-       pQA->nc->getData(ma_d, names[i] , j );
-       extr[rec++] = ma_d[0] ;
+       extr[rec++] = pQA->nc->getData(ma_d, names[i] , j );
+if( rec==7)
+  extr[6] += 100.;
     }
 
     // find number of extreme values exceeding N*sigma
@@ -533,12 +525,12 @@ Outlier::test(QA_Data *pQAD)
         outRec.clear();
         outVal.clear();
 
-        for( size_t j=0 ; j < currRecNum ; ++j )
+        for( size_t j=0 ; j < recNum ; ++j )
         {
           if( fabs(extr[j] - ave[i]) > n_sigma )
           {
             ++N[n] ;
-            outRec.push_back( prevRecEnd + j );
+            outRec.push_back( j );
             outVal.push_back( extr[j] );
           }
         }
@@ -571,7 +563,7 @@ Outlier::test(QA_Data *pQAD)
       if( outOptPrcnt > 0. )
       {
         double a =
-           static_cast<double>(N[n]) / static_cast<double>(currRecEnd) ;
+           static_cast<double>(N[n]) / static_cast<double>(recNum) ;
         isTooManyA =  a > outOptPrcnt ;
       }
       else