Various index count and aggregate fixes

Remove projects from count(keyed-limit(project(index)))
  A missing case from a switch statement meant projects weren't
  stripped when there was a keyed limit on the index read.

Reduce the generated code size for an index count/exists
  The numValid(size, ptr) member was never called for an index, so default
  it to throwing an exception in the base class, and clean up numValid(ptr)
  code.

CHOOSEN(LIMIT(index-read))
  The choosen was not being combined within the compound indexread if there
  was a limit.  It is however valid to do so since the choosen is done last.
  (A limit cannot be combined when there is an existing choosen.)
  This was unfortunate because it was frustrating  another optimization.
  count(index-read) > n being converted to count(choosen(index-read,n+1)) > n
  (This is temporarily retained, but will be removed in a subsequent commit).

IF(count(x) > 0, x, y) optimization
  Currently the intention is to perform a count-index on x before reading x.
  However if x contains a limit,skip it is going to need to read the entire
  result, so don't perform the count separately.

Don't generate count(index) if there is a skip limit

Optimize count(x) > 0 to exists(x)
  Instead of (count(x) > 0) becoming count(choosen(x, 1)) > 0 it is now
  transformed to exists(x) - which is either as efficient, or more
  efficient as the count representation.

Remove no_notexists since never created

Terminate hthor exists aggregate activity early

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlattr.cpp

@@ -228,7 +228,6 @@ unsigned getOperatorMetaFlags(node_operator op)
 //Aggregate operators
     case no_count:
     case no_exists:
-    case no_notexists:
     case no_max:
     case no_min:
     case no_sum:
@@ -238,7 +237,6 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_correlation:
     case no_countgroup:
     case no_existsgroup:
-    case no_notexistsgroup:
     case no_maxgroup:
     case no_mingroup:
     case no_sumgroup:
@@ -614,7 +612,7 @@ unsigned getOperatorMetaFlags(node_operator op)
     case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
     case no_unused40: case no_unused41: case no_unused42: case no_unused43: case no_unused44: case no_unused45: case no_unused46: case no_unused47: case no_unused48: case no_unused49:
     case no_unused50: case no_unused52:
-    case no_unused80:
+    case no_unused80: case no_unused82: case no_unused83:
     case no_is_null:
     case no_position:
     case no_current_time:

ecl/hql/hqlexpr.cpp

@@ -976,9 +976,7 @@ const char *getOpString(node_operator op)
     case no_mapto: return "=>";
     case no_constant: return "<constant>";
     case no_field: return "<field>";
-    case no_notexists: return "NOT EXISTS";
     case no_exists: case no_existslist: return "EXISTS";
-    case no_notexistsgroup: return "NOT EXISTS";
     case no_existsgroup: return "EXISTS";
     case no_select: return ".";
     case no_table: return "DATASET";
@@ -1478,8 +1476,6 @@ node_operator getInverseOp(node_operator op)
     case no_in: return no_notin;
     case no_notbetween: return no_between;
     case no_between: return no_notbetween;
-    case no_notexists: return no_exists;
-    case no_exists: return no_notexists;
 //  case no_notwithin: return no_within;
 //  case no_within: return no_notwithin;
     default:
@@ -14676,7 +14672,6 @@ IHqlExpression * convertToSimpleAggregate(IHqlExpression * expr)
     case no_maxgroup:       newop = no_max; numArgs = 1; break;
     case no_sumgroup:       newop = no_sum; numArgs = 1; break;
     case no_existsgroup:    newop = no_exists; break;
-    case no_notexistsgroup: newop = no_notexists; break;
     default: 
         return NULL;
     }
@@ -14715,7 +14710,6 @@ IHqlExpression * queryAggregateFilter(IHqlExpression * expr)
     {
     case no_countgroup:
     case no_existsgroup:
-    case no_notexistsgroup:
         return queryRealChild(expr, 0);
     case no_sumgroup:
     case no_vargroup:
@@ -14774,7 +14768,6 @@ node_operator querySingleAggregate(IHqlExpression * expr, bool canFilterArg, boo
             switch (curOp)
             {
             case no_existsgroup:
-            case no_notexistsgroup:
             case no_countgroup:
                 break;
             default:

ecl/hql/hqlexpr.hpp

@@ -224,7 +224,7 @@ enum _node_operator {
         no_comma,
         no_count,
         no_countgroup,
-        no_notexists,
+    no_unused82,
         no_exists,
         no_within,
         no_notwithin,
@@ -535,7 +535,7 @@ enum _node_operator {
         no_outputscalar,
         no_matchunicode,
         no_pat_validate,
-        no_notexistsgroup,
+   no_unused83,
         no_existsgroup,
         no_pat_use,
         no_unused13,
@@ -1743,8 +1743,7 @@ extern HQL_API void gatherWarnings(IErrorReceiver * errs, IHqlExpression * expr)
     case no_max:            \
     case no_min:            \
     case no_ave:            \
-    case no_exists:         \
-    case no_notexists
+    case no_exists
 
 #define NO_AGGREGATEGROUP   \
          no_countgroup:         \
@@ -1755,8 +1754,7 @@ extern HQL_API void gatherWarnings(IErrorReceiver * errs, IHqlExpression * expr)
     case no_maxgroup:           \
     case no_mingroup:           \
     case no_avegroup:           \
-    case no_existsgroup:        \
-    case no_notexistsgroup
+    case no_existsgroup
 
 extern HQL_API ITypeInfo * getTypedefType(IHqlExpression * expr);
 

ecl/hql/hqlfold.cpp

@@ -3786,9 +3786,6 @@ IHqlExpression * NullFolderMixin::queryOptimizeAggregateInline(IHqlExpression *
     case no_existsgroup:
         value.setown(createConstant(numRows != 0));
         break;
-    case no_notexistsgroup:
-        value.setown(createConstant(numRows == 0));
-        break;
     case no_countgroup:
         {
             ITypeInfo * type = assign->queryChild(0)->queryType();
@@ -4382,7 +4379,6 @@ IHqlExpression * CExprFolderTransformer::doFoldTransformed(IHqlExpression * unfo
             break;
         }
     case no_exists:
-    case no_notexists:
         {
             IHqlExpression * child = expr->queryChild(0);
             node_operator childOp = child->getOperator();
@@ -4392,15 +4388,15 @@ IHqlExpression * CExprFolderTransformer::doFoldTransformed(IHqlExpression * unfo
                 if (isPureInlineDataset(child))
                 {
                     bool hasChildren = (child->queryChild(0)->numChildren() != 0);
-                    return createConstant((op == no_exists) ? hasChildren : !hasChildren);
+                    return createConstant(hasChildren);
                 }
                 break;
 #if 0
             case no_addfiles:
                 {
                     OwnedHqlExpr lhs = replaceChild(expr, 0, child->queryChild(0));
                     OwnedHqlExpr rhs = replaceChild(expr, 0, child->queryChild(1));
-                    return createValue((op == no_exists) ? no_or : no_add, expr->getType(), LINK(lhs), LINK(rhs));
+                    return createValue(no_or, expr->getType(), LINK(lhs), LINK(rhs));
                 }
             case no_if:
                 {
@@ -5186,7 +5182,6 @@ IHqlExpression * CExprFolderTransformer::createTransformed(IHqlExpression * expr
             case no_min:
             case no_sum:
             case no_exists:
-            case no_notexists:
             case no_ave:
                 //Could implement this on a temp table, or at least count...
                 //not sufficient to just fix these, because functions of these also fail.

ecl/hql/hqlgram2.cpp

@@ -7302,7 +7302,6 @@ void HqlGram::checkConditionalAggregates(_ATOM name, IHqlExpression * value, con
         break;
     case no_existsgroup:
     case no_countgroup:
-    case no_notexistsgroup:
         cond = queryRealChild(value, 0);
         break;
     case no_covargroup:

ecl/hql/hqlopt.cpp

@@ -495,6 +495,7 @@ IHqlExpression * CTreeOptimizer::optimizeAggregateUnsharedDataset(IHqlExpression
         break;
     case no_compound_indexread:
     case no_compound_diskread:
+    case no_keyedlimit:
         break;
     case no_limit:
         if (expr->hasProperty(onFailAtom))

ecl/hql/hqlutil.cpp

@@ -3928,7 +3928,6 @@ extern HQL_API IHqlExpression * convertScalarAggregateToDataset(IHqlExpression *
     case no_max:   newop = no_maxgroup; break;
     case no_sum:   newop = no_sumgroup; break;
     case no_exists:newop = no_existsgroup; break;
-    case no_notexists:  newop = no_notexistsgroup; break;
     case no_variance:   newop = no_vargroup; break;
     case no_covariance: newop = no_covargroup; break;
     case no_correlation:newop = no_corrgroup; break;

ecl/hqlcpp/hqlcpp.cpp

@@ -2486,7 +2486,6 @@ void HqlCppTranslator::buildExprAssign(BuildCtx & ctx, const CHqlBoundTarget & t
     case no_min:
     case no_sum:
     case no_exists:
-    case no_notexists:
         doBuildAssignAggregate(ctx, target, expr);
         break;
     case no_getenv:
@@ -2799,7 +2798,6 @@ void HqlCppTranslator::buildExpr(BuildCtx & ctx, IHqlExpression * expr, CHqlBoun
             doBuildExprAggregate(ctx, expr, tgt);
         return;
     case no_exists:
-    case no_notexists:
         if (!(expr->isPure() && ctx.getMatchExpr(expr, tgt)))
             doBuildExprExists(ctx, expr, tgt);
         return;

ecl/hqlcpp/hqlcppds.cpp

@@ -631,7 +631,6 @@ void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoun
     switch (op)
     {
     case no_exists:
-    case no_notexists:
         {
             OwnedHqlExpr optimized = queryOptimizedExists(ctx, expr, dataset);
             if (optimized)
@@ -680,17 +679,25 @@ void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoun
     //If no_if or no_addfiles has been optimized above then the selector for the argument will have changed => map it.
     if (arg && (dataset != oldDataset))
         arg.setown(replaceSelector(arg, oldDataset, dataset));
-    bool needToBreak = (op == no_exists || op == no_notexists);
+
+    bool needToBreak = (op == no_exists);
+    if (needToBreak)
+    {
+        //if it can have at most one row (fairly strange code!) then don't add a break
+        //unless it was deliberately a choosen to restrict the number of iterations.
+        if (hasNoMoreRowsThan(dataset, 1) && (dataset->getOperator() != no_choosen))
+            needToBreak = false;
+    }
 
     BuildCtx loopctx(ctx);
     buildDatasetIterate(loopctx, dataset, needToBreak);
 
     switch (op)
     {
     case no_exists:
-    case no_notexists:
-        buildExprAssign(loopctx, target, queryBoolExpr(op==no_exists));
-        loopctx.addBreak();
+        buildExprAssign(loopctx, target, queryBoolExpr(true));
+        if (needToBreak)
+            loopctx.addBreak();
         break;
     case no_count:
         {
@@ -748,7 +755,6 @@ bool assignAggregateDirect(const CHqlBoundTarget & target, IHqlExpression * expr
             break;
         //fall through
     case no_exists:
-    case no_notexists:
     case no_count:
         if (target.expr->getOperator() != no_variable)
             return false;
@@ -784,8 +790,7 @@ void HqlCppTranslator::doBuildAssignAggregate(BuildCtx & ctx, const CHqlBoundTar
         switch (op)
         {
         case no_exists:
-        case no_notexists:
-            buildExprAssign(ctx, target, queryBoolExpr(op==no_notexists));
+            buildExprAssign(ctx, target, queryBoolExpr(false));
             break;
         default:
             {
@@ -3401,9 +3406,6 @@ void HqlCppTranslator::doBuildRowAssignAggregateClear(BuildCtx & ctx, IReference
         case no_existsgroup:
             curTarget->buildClear(ctx, 0);
             break;
-        case no_notexistsgroup:
-            curTarget->set(ctx, queryBoolExpr(true));
-            break;
         default:
             if (src->isConstant())
                 curTarget->set(ctx, src);
@@ -3484,11 +3486,10 @@ void HqlCppTranslator::doBuildRowAssignAggregateNext(BuildCtx & ctx, IReferenceS
             }
             break;
         case no_existsgroup:
-        case no_notexistsgroup:
             assertex(!(arg && isVariableOffset));
             if (arg)
                 buildFilter(condctx, arg);
-            curTarget->set(condctx, queryBoolExpr(srcOp == no_existsgroup));
+            curTarget->set(condctx, queryBoolExpr(true));
             if (isSingleExists)
                 condctx.addBreak();
             break;
@@ -3535,7 +3536,6 @@ void HqlCppTranslator::doBuildRowAssignAggregate(BuildCtx & ctx, IReferenceSelec
             isSingleExists = false;
             break;
         case no_existsgroup:
-        case no_notexistsgroup:
             break;
         case no_mingroup:
             isSingleExists = false;

ecl/hqlcpp/hqlcse.cpp

@@ -177,7 +177,6 @@ bool CseSpotterInfo::useInverseForAlias()
     case no_ne:
     case no_notin:
     case no_notbetween:
-    case no_notexists:
         return inverse->worthAliasingOnOwn();
     }
 
@@ -188,7 +187,6 @@ bool CseSpotterInfo::useInverseForAlias()
     case no_ne:
     case no_notin:
     case no_notbetween:
-    case no_notexists:
         return !worthAliasingOnOwn();
     }
     return op > invOp;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -11841,9 +11841,6 @@ void HqlCppTranslator::doBuildAggregateClearFunc(BuildCtx & ctx, IHqlExpression
         case no_existsgroup:
             buildClear(funcctx, target);
             break;
-        case no_notexistsgroup:
-            buildAssign(funcctx, target, queryBoolExpr(true));
-            break;
         default:
             if (src->isConstant())
                 buildAssign(funcctx, target, src);
@@ -11984,15 +11981,14 @@ void HqlCppTranslator::doBuildAggregateProcessTransform(BuildCtx & ctx, BoundRow
             }
             break;
         case no_existsgroup:
-        case no_notexistsgroup:
             assertex(!(arg && isVariableOffset));
             cond = arg;
             if (cond || !alwaysNextRow)
             {
                 //The assign is conditional because unconditionally it is done in the AggregateFirst
                 if (cond)
                     buildFilter(condctx, cond);
-                buildAssign(condctx, target, queryBoolExpr(srcOp == no_existsgroup));
+                buildAssign(condctx, target, queryBoolExpr(true));
             }
             break;
         default:
@@ -12083,13 +12079,6 @@ void HqlCppTranslator::doBuildAggregateMergeFunc(BuildCtx & ctx, IHqlExpression
                 buildAssign(condctx, target, queryBoolExpr(true));
                 break;
             }
-        case no_notexistsgroup:
-            {
-                BuildCtx condctx(funcctx);
-                buildFilter(condctx, target);
-                buildAssign(condctx, target, src);
-            }
-            break;
         default:
             //already filled in and wouldn't be legal to have an expression in this case anyway...
             break;