New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DATASET (N, transform(COUNTER)) #1285

Merged
merged 1 commit into from Jan 19, 2012
Jump to file or symbol
Failed to load files and symbols.
+167 −24
Diff settings

Always

Just for now

View
@@ -605,8 +605,9 @@ unsigned getOperatorMetaFlags(node_operator op)
case no_bound_type:
case no_mix:
case no_persist_check:
case no_dataset_from_transform:
case no_unused1: case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
@@ -2598,6 +2599,16 @@ IHqlExpression * calcRowInformation(IHqlExpression * expr)
info.setRange(minValue, maxValue);
break;
}
case no_dataset_from_transform:
{
// only if the count is a constant value
IHqlExpression * count = expr->queryChild(0);
IValue * value = count->queryValue();
if (value)
info.setN(value->getIntValue());
// leave it be, if it's a constant expression or a variable
break;
}
case no_null:
info.setN(expr->isDatarow() ? 1 : 0);
break;
View
@@ -1075,6 +1075,7 @@ const char *getOpString(node_operator op)
case no_transformebcdic: return "EBCDIC";
case no_transformascii: return "ASCII";
case no_hqlproject: return "PROJECT";
case no_dataset_from_transform: return "DATASET";
case no_newtransform: return "NEWTRANSFORM";
case no_transform: return "TRANSFORM";
case no_attr: return "no_attr";
@@ -1424,7 +1425,7 @@ const char *getOpString(node_operator op)
case no_debug_option_value: return "__DEBUG__";
case no_dataset_alias: return "TABLE";
case no_unused1: case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
case no_unused2: case no_unused3: case no_unused4: case no_unused5: case no_unused6:
case no_unused13: case no_unused14: case no_unused15: case no_unused17: case no_unused18: case no_unused19:
case no_unused20: case no_unused21: case no_unused22: case no_unused23: case no_unused24: case no_unused25: case no_unused26: case no_unused27: case no_unused28: case no_unused29:
case no_unused30: case no_unused31: case no_unused32: case no_unused33: case no_unused34: case no_unused35: case no_unused36: case no_unused37: case no_unused38:
@@ -1772,6 +1773,7 @@ childDatasetType getChildDatasetType(IHqlExpression * expr)
case no_definesideeffect:
case no_callsideeffect:
case no_fromxml:
case no_dataset_from_transform:
return childdataset_none;
case no_group:
case no_grouped:
@@ -2167,6 +2169,7 @@ inline unsigned doGetNumChildTables(IHqlExpression * dataset)
case no_definesideeffect:
case no_callsideeffect:
case no_fromxml:
case no_dataset_from_transform:
return 0;
case no_delayedselect:
case no_libraryselect:
@@ -2477,6 +2480,7 @@ bool definesColumnList(IHqlExpression * dataset)
case no_normalizegroup:
case no_cogroup:
case no_dataset_alias:
case no_dataset_from_transform:
return true;
case no_select:
case no_field:
@@ -2574,6 +2578,7 @@ IHqlExpression * queryNewColumnProvider(IHqlExpression * expr)
case no_transformebcdic:
case no_transformascii:
case no_hqlproject:
case no_dataset_from_transform:
case no_keyindex:
case no_projectrow:
case no_iterate:
@@ -10321,6 +10326,7 @@ IHqlExpression *createDataset(node_operator op, HqlExprArray & parms)
assertex(parms.ordinality()>1); // should have a count or a length
break;
case no_inlinetable:
case no_dataset_from_transform:
case no_xmlproject:
case no_temptable:
case no_id2blob:
View
@@ -352,7 +352,7 @@ enum _node_operator {
no_unused22,
no_unused23,
no_unused24,
no_unused1,
no_dataset_from_transform,
no_unused2,
no_unknown,
no_unused3,
View
@@ -3532,6 +3532,10 @@ IHqlExpression * NullFolderMixin::foldNullDataset(IHqlExpression * expr)
if (expr->queryChild(0)->numChildren() == 0)
return replaceWithNull(expr);
break;
case no_dataset_from_transform:
if (isZero(expr->queryChild(0)))
return replaceWithNull(expr);
break;
case no_temptable:
{
IHqlExpression * values = expr->queryChild(0);
@@ -5336,6 +5340,7 @@ HqlConstantPercolator * CExprFolderTransformer::gatherConstants(IHqlExpression *
case no_nwayjoin:
case no_projectrow:
case no_createrow:
case no_dataset_from_transform:
{
IHqlExpression * transform = queryNewColumnProvider(expr);
exprMapping.setown(HqlConstantPercolator::extractConstantMapping(transform));
View
@@ -1,4 +1,3 @@
//##############################################################################
//
// Copyright (C) 2011 HPCC Systems.
//
@@ -6280,7 +6279,7 @@ beginCounterScope
:
{
parser->counterStack.append(* new OwnedHqlExprItem);
$$.clear();
$$.clear();
}
;
@@ -7887,14 +7886,17 @@ simpleDataSet
$$.setExpr(dataset.getClear());
$$.setPosition($1);
}
| DATASET '(' thorFilenameOrList ',' dsRecordDef ',' mode optDsOptions dsEnd
| DATASET '(' thorFilenameOrList ',' beginCounterScope dsRecordDef endCounterScope ',' mode optDsOptions dsEnd
{
parser->warnIfRecordPacked($5);
parser->transferOptions($3, $8);
OwnedHqlExpr counter = $7.queryExpr();
if (counter)
parser->reportError(ERR_ILL_HERE,$6,"Not expecting COUNTER for DATASET");
parser->warnIfRecordPacked($6);
parser->transferOptions($3, $10);
parser->normalizeExpression($3, type_string, false);
OwnedHqlExpr mode = $7.getExpr();
OwnedHqlExpr options = $8.getExpr();
OwnedHqlExpr mode = $9.getExpr();
OwnedHqlExpr options = $10.getExpr();
OwnedHqlExpr filename = $3.getExpr();
if (mode->getOperator() == no_comma)
{
@@ -7912,18 +7914,21 @@ simpleDataSet
filename.setown(createValue(no_assertconstant, filename->getType(), LINK(filename->queryChild(0))));
options.setown(createComma(options.getClear(), createAttribute(localUploadAtom)));
}
IHqlExpression * dataset = createNewDataset(filename.getClear(), $5.getExpr(), mode.getClear(), NULL, NULL, options.getClear());
parser->checkValidRecordMode(dataset, $4, $7);
IHqlExpression * dataset = createNewDataset(filename.getClear(), $6.getExpr(), mode.getClear(), NULL, NULL, options.getClear());
parser->checkValidRecordMode(dataset, $4, $9);
$$.setExpr(dataset);
$$.setPosition($1);
}
| DATASET '(' thorFilenameOrList ',' simpleType optDsOptions dsEnd
| DATASET '(' thorFilenameOrList ',' beginCounterScope simpleType endCounterScope optDsOptions dsEnd
{
parser->transferOptions($3, $6);
OwnedHqlExpr counter = $7.queryExpr();
if (counter)
parser->reportError(ERR_ILL_HERE,$6,"Not expecting COUNTER for DATASET");
parser->transferOptions($3, $8);
parser->normalizeExpression($3, type_string, false);
OwnedHqlExpr options = $6.getExpr();
OwnedITypeInfo type = $5.getType();
OwnedHqlExpr options = $8.getExpr();
OwnedITypeInfo type = $6.getType();
OwnedHqlExpr filename = $3.getExpr();
OwnedHqlExpr option;
switch (type->getTypeCode())
@@ -7953,7 +7958,7 @@ simpleDataSet
options.setown(createComma(options.getClear(), createAttribute(localUploadAtom)));
}
IHqlExpression * dataset = createNewDataset(filename.getClear(), record.getClear(), mode.getClear(), NULL, NULL, options.getClear());
parser->checkValidRecordMode(dataset, $4, $7);
parser->checkValidRecordMode(dataset, $4, $9);
$$.setExpr(dataset, $1);
}
| DATASET '(' dataSet ',' thorFilenameOrList ',' mode optDsOptions dsEnd
@@ -7989,12 +7994,15 @@ simpleDataSet
$$.setExpr(createDataset(no_inlinetable, createValue(no_transformlist, makeNullType(), values), LINK(record)));
$$.setPosition($1);
}
| DATASET '(' thorFilenameOrList ',' recordDef ')'
| DATASET '(' thorFilenameOrList ',' beginCounterScope recordDef endCounterScope ')'
{
//NB: $3 is required to be a list, but uses thorfilename production to work around a s/r error
OwnedHqlExpr counter = $7.queryExpr();
if (counter)
parser->reportError(ERR_ILL_HERE,$6,"Not expecting COUNTER for DATASET");
parser->normalizeExpression($3, type_set, false);
$$.setExpr(parser->createDatasetFromList($3, $5), $1);
$$.setExpr(parser->createDatasetFromList($3, $6), $1);
}
| DATASET '(' WORKUNIT '(' expression ',' expression ')' ',' recordDef ')'
{
@@ -8026,6 +8034,16 @@ simpleDataSet
$$.setExpr(createDataset(no_workunit_dataset, $8.getExpr(), arg));
$$.setPosition($1);
}
| DATASET '(' thorFilenameOrList ',' beginCounterScope transform endCounterScope ')'
{
// TODO: use DISTRIBUTED flag
parser->normalizeExpression($3, type_int, false);
IHqlExpression * counter = $7.getExpr();
if (counter)
counter = createAttribute(_countProject_Atom, counter);
$$.setExpr(createDataset(no_dataset_from_transform, $3.getExpr(), createComma($6.getExpr(), counter)));
$$.setPosition($1);
}
| ENTH '(' dataSet ',' expression optCommonAttrs ')'
{
parser->normalizeExpression($5, type_numeric, false);
View
@@ -1330,6 +1330,7 @@ public:
ABoundActivity * doBuildActivityCombineGroup(BuildCtx & ctx, IHqlExpression * expr);
ABoundActivity * doBuildActivityCompoundSelectNew(BuildCtx & ctx, IHqlExpression * expr);
ABoundActivity * doBuildActivityConcat(BuildCtx & ctx, IHqlExpression * expr);
ABoundActivity * doBuildActivityCountTransform(BuildCtx & ctx, IHqlExpression * expr);
ABoundActivity * doBuildActivityCreateRow(BuildCtx & ctx, IHqlExpression * expr, bool isDataset);
ABoundActivity * doBuildActivityXmlRead(BuildCtx & ctx, IHqlExpression * expr);
ABoundActivity * doBuildActivityDedup(BuildCtx & ctx, IHqlExpression * expr);
View
@@ -5961,6 +5961,9 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
case no_compound_inline:
result = doBuildActivityChildDataset(ctx, expr->queryChild(0));
break;
case no_dataset_from_transform:
result = doBuildActivityCountTransform(ctx, expr);
break;
case no_table:
result = doBuildActivityTable(ctx, expr);
break;
@@ -15925,6 +15928,42 @@ ABoundActivity * HqlCppTranslator::doBuildActivityInlineTable(BuildCtx & ctx, IH
//---------------------------------------------------------------------------
ABoundActivity * HqlCppTranslator::doBuildActivityCountTransform(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * count = expr->queryChild(0);
IHqlExpression * transform = queryNewColumnProvider(expr);
IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
// Overriding IHThorTempTableArg
Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKtemptable, expr,"TempTable");
buildActivityFramework(instance);
buildInstancePrefix(instance);
// size32_t getRow()
BuildCtx funcctx(instance->startctx);
funcctx.addQuotedCompound("virtual size32_t getRow(ARowBuilder & crSelf, unsigned row)");
ensureRowAllocated(funcctx, "crSelf");
BoundRow * selfCursor = bindSelf(funcctx, instance->dataset, "crSelf");
IHqlExpression * self = selfCursor->querySelector();
associateCounter(funcctx, counter, "row");
// FIXME: this should be fixed in the engine
funcctx.addQuoted("if (row == numRows()) return 0;");
buildTransformBody(funcctx, transform, NULL, NULL, instance->dataset, self);
// unsigned numRows() - count is guaranteed by lexer
doBuildUnsignedFunction(instance->startctx, "numRows", count);
// bool isConstant() - default is true
if (!isConstantTransform(transform))
doBuildBoolFunction(instance->startctx, "isConstant", false);
buildInstanceSuffix(instance);
return instance->getBoundActivity();
}
//---------------------------------------------------------------------------
void HqlCppTranslator::buildHTTPtoXml(BuildCtx & ctx)
{
BuildCtx funcctx(ctx);
@@ -0,0 +1,51 @@
/*##############################################################################
Copyright (C) 2011 HPCC Systems.
All rights reserved. This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
############################################################################## */
C := 5 : stored('C');
r := record
unsigned i;
end;
r t1(unsigned value) := transform
SELF.i := value * 10;
end;
r t2() := transform
SELF.i := 10;
end;
// zero
output(true);
ds := DATASET(0, t1(COUNTER));
output(ds);
// plain
output(true);
ds10 := DATASET(10, t1(COUNTER));
output(ds10);
// expr
output(true);
ds50 := DATASET(5 * 10, t1(COUNTER));
output(ds50);
// variable
output(true);
ds5 := DATASET(C, t2());
output(ds5);
View
@@ -30,21 +30,25 @@
syntax="syntax: $0 [-t target_dir] [-c compare_dir] [-i include_dir] [-e eclcc binary] -n [parallel processes] [-r (no run, just compare)]"
## Create Makefile (git doesn't like tabs)
echo "FILES=\$(shell echo *.ecl)" > Makefile
echo "LOGS=\$(FILES:%.ecl=\$(target_dir)/%.log)" >> Makefile
echo "FILES=\$(shell echo *.ecl*)" > Makefile
echo "LOGS_=\$(FILES:%.ecl=\$(target_dir)/%.log)" >> Makefile
echo "LOGS=\$(LOGS_:%.eclxml=\$(target_dir)/%.log)" >> Makefile
echo >> Makefile
echo "all: \$(LOGS)" >> Makefile
echo >> Makefile
echo "\$(target_dir)/%.log: %.ecl" >> Makefile
echo -e "\t\$(eclcc) \$(flags) $^" >> Makefile
echo >> Makefile
echo "\$(target_dir)/%.log: %.eclxml" >> Makefile
echo -e "\t\$(eclcc) \$(flags) $^" >> Makefile
## Default arguments
target_dir=run_$$
compare_dir=
include_dir=
compare_only=0
eclcc=`which eclcc`
np=10
np=`grep -c processor /proc/cpuinfo`
## Get cmd line options (overrite default args)
if [[ $* != '' ]]; then
@@ -97,9 +101,16 @@ if [[ $compare_dir ]]; then
echo " ++ No target dir to compare"
exit 1
fi
echo "* Comparing to Golden Standard"
echo "* Comparing to $compare_dir"
echo
diff -I $compare_dir -I $target_dir -I '\d* ms' -q $compare_dir $target_dir
if [[ $compare_only = 0 ]]; then
quick=-q
fi
diff -I $compare_dir \
-I $target_dir \
-I '\d* ms' \
-I 'at \/.*\(\d*\)$' \
$quick $compare_dir $target_dir
fi
# Confirmation
@@ -636,6 +636,7 @@ interface ICsvToRowTransformer : public IInterface
// Activity index: Class name = s/TAK(.*)/IHThor$1Arg/, with $1 using camel case
enum ThorActivityKind
{
//This list cannot be reordered - unless all workunits are invalidated...
ProTip! Use n and p to navigate between commits in a pull request.