Skip to content

Commit c8863f4

Browse files
hbutanihbutani
committed
NPath ResultExpr parser
1 parent 701fc12 commit c8863f4

File tree

3 files changed

+281
-5
lines changed

3 files changed

+281
-5
lines changed

‎windowing/src/main/java/com/sap/hadoop/windowing/functions2/table/npath/NPathUtils.java

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
package com.sap.hadoop.windowing.functions2.table.npath;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
6+
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
7+
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
8+
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
9+
import org.apache.hadoop.hive.serde2.objectinspector.StandardListObjectInspector;
10+
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
11+
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
12+
13+
import com.sap.hadoop.windowing.runtime2.PartitionIterator;
14+
15+
public class NPathUtils
16+
{
17+
public static final String PATHATTR_NAME = "tpath";
18+
/*
19+
* add array<struct> to the list of columns
20+
*/
21+
public static ObjectInspector createSelectListInputOI(StructObjectInspector inputOI)
22+
{
23+
List<? extends StructField> fields = inputOI.getAllStructFieldRefs();
24+
ArrayList<ObjectInspector> selectListFieldOIs = new ArrayList<ObjectInspector>();
25+
ArrayList<String> selectListFieldNames = new ArrayList<String>();
26+
StandardListObjectInspector pathAttrOI = ObjectInspectorFactory.getStandardListObjectInspector(
27+
ObjectInspectorUtils.getStandardObjectInspector(inputOI));
28+
for(StructField f : fields)
29+
{
30+
selectListFieldOIs.add(ObjectInspectorUtils.getStandardObjectInspector(f.getFieldObjectInspector()));
31+
selectListFieldNames.add(f.getFieldName());
32+
}
33+
selectListFieldOIs.add(pathAttrOI);
34+
selectListFieldNames.add(PATHATTR_NAME);
35+
return ObjectInspectorFactory.getStandardStructObjectInspector(selectListFieldNames, selectListFieldOIs);
36+
}
37+
38+
public static Object getSelectListInput(Object currRow, ObjectInspector rowOI, PartitionIterator<Object> pItr, int sz)
39+
{
40+
ArrayList<Object> oRow = new ArrayList<Object>();
41+
List<?> currRowAsStdObject = (List<?>) ObjectInspectorUtils.copyToStandardObject(currRow, rowOI);
42+
oRow.addAll(currRowAsStdObject);
43+
oRow.add(getPath(currRow, rowOI, pItr, sz));
44+
return oRow;
45+
}
46+
47+
public static ArrayList<Object> getPath(Object currRow, ObjectInspector rowOI, PartitionIterator<Object> pItr, int sz)
48+
{
49+
int idx = pItr.getIndex() - 1;
50+
ArrayList<Object> path = new ArrayList<Object>();
51+
path.add(ObjectInspectorUtils.copyToStandardObject(currRow, rowOI));
52+
int pSz = 1;
53+
54+
while(pSz < sz && pItr.hasNext())
55+
{
56+
currRow = pItr.next();
57+
path.add(ObjectInspectorUtils.copyToStandardObject(currRow, rowOI));
58+
pSz++;
59+
}
60+
pItr.resetToIndex(idx);
61+
return path;
62+
}
63+
}

‎windowing/src/main/java/com/sap/hadoop/windowing/functions2/table/npath/ResultExpressionParser.java

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
package com.sap.hadoop.windowing.functions2.table.npath;
2+
3+
import java.util.ArrayList;
4+
import java.util.Iterator;
5+
6+
import org.antlr.runtime.ANTLRStringStream;
7+
import org.antlr.runtime.CommonTokenStream;
8+
import org.antlr.runtime.tree.CommonTree;
9+
import org.antlr.runtime.tree.CommonTreeNodeStream;
10+
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
11+
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
12+
import org.apache.hadoop.hive.ql.metadata.HiveException;
13+
import org.apache.hadoop.hive.ql.parse.ASTNode;
14+
import org.apache.hadoop.hive.ql.parse.RowResolver;
15+
import org.apache.hadoop.hive.ql.parse.TypeCheckCtx;
16+
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
17+
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
18+
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
19+
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
20+
21+
import com.sap.hadoop.HiveUtils;
22+
import com.sap.hadoop.windowing.WindowingException;
23+
import com.sap.hadoop.windowing.parser.QSpecBuilder2;
24+
import com.sap.hadoop.windowing.parser.Windowing2Lexer;
25+
import com.sap.hadoop.windowing.parser.Windowing2Parser;
26+
import com.sap.hadoop.windowing.query2.specification.SelectSpec;
27+
import com.sap.hadoop.windowing.query2.translate.TranslateUtils;
28+
29+
/*
30+
* ResultExpression is a Select List with the following variation:
31+
* - the select keyword is optional. The parser checks if the expression doesn't start with select; if not it prefixes it.
32+
* - Window Fn clauses are not permitted.
33+
* - expressions can operate on the input columns plus the psuedo column 'path' which is array of structs. The shape of the struct is
34+
* the same as the input.
35+
*
36+
*/
37+
public class ResultExpressionParser
38+
{
39+
String resultExprString;
40+
41+
StructObjectInspector selectListInputOI;
42+
RowResolver selectListInputRowResolver;
43+
TypeCheckCtx selectListInputTypeCheckCtx;
44+
45+
SelectSpec selectSpec;
46+
47+
ArrayList<ExprNodeEvaluator> selectListExprEvaluators;
48+
ArrayList<ObjectInspector> selectListExprOIs;
49+
ObjectInspector selectListOutputOI;
50+
51+
public ResultExpressionParser(String resultExprString, StructObjectInspector selectListInputOI)
52+
{
53+
this.resultExprString = resultExprString;
54+
this.selectListInputOI = selectListInputOI;
55+
}
56+
57+
public void translate() throws WindowingException
58+
{
59+
setupSelectListInputInfo();
60+
fixResultExprString();
61+
parse();
62+
validateSelectExpr();
63+
buildSelectListEvaluators();
64+
}
65+
66+
public ArrayList<ExprNodeEvaluator> getSelectListExprEvaluators()
67+
{
68+
return selectListExprEvaluators;
69+
}
70+
71+
public StructObjectInspector getSelectListOutputOI()
72+
{
73+
return (StructObjectInspector) selectListOutputOI;
74+
}
75+
76+
private void buildSelectListEvaluators() throws WindowingException
77+
{
78+
selectListExprEvaluators = new ArrayList<ExprNodeEvaluator>();
79+
selectListExprOIs = new ArrayList<ObjectInspector>();
80+
ArrayList<String> selectListExprNames = new ArrayList<String>();
81+
int i = 0;
82+
Iterator<Object> it = selectSpec.getColumnListAndAlias();
83+
while(it.hasNext())
84+
{
85+
Object[] selectColDetails = (Object[]) it.next();
86+
String selectColName = (String) selectColDetails[1];
87+
ASTNode selectColumnNode = (ASTNode) selectColDetails[2];
88+
ExprNodeDesc selectColumnExprNode = TranslateUtils.buildExprNode(selectColumnNode, selectListInputTypeCheckCtx);
89+
ExprNodeEvaluator selectColumnExprEval = ExprNodeEvaluatorFactory.get(selectColumnExprNode);
90+
ObjectInspector selectColumnOI = null;
91+
try
92+
{
93+
selectColumnOI = selectColumnExprEval.initialize(selectListInputOI);
94+
}
95+
catch (HiveException he)
96+
{
97+
throw new WindowingException(he);
98+
}
99+
100+
selectColName = selectColName == null ? "npath_col_" + i : selectColName;
101+
102+
selectListExprEvaluators.add(selectColumnExprEval);
103+
selectListExprOIs.add(selectColumnOI);
104+
selectListExprNames.add(selectColName);
105+
i++;
106+
}
107+
108+
selectListOutputOI = ObjectInspectorFactory.getStandardStructObjectInspector(selectListExprNames, selectListExprOIs);
109+
}
110+
111+
private void setupSelectListInputInfo() throws WindowingException
112+
{
113+
selectListInputRowResolver = HiveUtils.getRowResolver("npathInput", selectListInputOI);
114+
selectListInputTypeCheckCtx = new TypeCheckCtx(selectListInputRowResolver);
115+
selectListInputTypeCheckCtx.setUnparseTranslator(null);
116+
}
117+
118+
private void fixResultExprString()
119+
{
120+
String r = resultExprString.trim();
121+
String prefix = r.substring(0, 6);
122+
if ( !prefix.toLowerCase().equals("select"))
123+
{
124+
r = "select " + r;
125+
}
126+
resultExprString = r;
127+
}
128+
129+
private void parse() throws WindowingException
130+
{
131+
Windowing2Lexer lexer;
132+
CommonTokenStream tokens;
133+
Windowing2Parser parser = null;
134+
CommonTree t;
135+
CommonTreeNodeStream nodes;
136+
QSpecBuilder2 qSpecBldr = null;
137+
String err;
138+
139+
try
140+
{
141+
lexer = new Windowing2Lexer(new ANTLRStringStream(resultExprString));
142+
tokens = new CommonTokenStream(lexer);
143+
parser = new Windowing2Parser(tokens);
144+
parser.setTreeAdaptor(TranslateUtils.adaptor);
145+
t = (CommonTree) parser.select().getTree();
146+
147+
err = parser.getWindowingParseErrors();
148+
if ( err != null )
149+
{
150+
throw new WindowingException(err);
151+
}
152+
}
153+
catch(Throwable te)
154+
{
155+
err = parser.getWindowingParseErrors();
156+
if ( err != null )
157+
{
158+
throw new WindowingException(err);
159+
}
160+
throw new WindowingException("Parse Error:" + te.toString(), te);
161+
}
162+
163+
TranslateUtils.unescapeStringLiterals((ASTNode) t);
164+
165+
try
166+
{
167+
168+
nodes = new CommonTreeNodeStream(t);
169+
nodes.setTokenStream(tokens);
170+
qSpecBldr = new QSpecBuilder2(nodes);
171+
selectSpec = qSpecBldr.select();
172+
173+
err = qSpecBldr.getWindowingParseErrors();
174+
if ( err != null )
175+
{
176+
throw new WindowingException(err);
177+
}
178+
179+
}
180+
catch(Throwable te)
181+
{
182+
err = qSpecBldr.getWindowingParseErrors();
183+
if ( err != null )
184+
{
185+
throw new WindowingException(err);
186+
}
187+
throw new WindowingException("Parse Error:" + te.toString(), te);
188+
}
189+
}
190+
191+
private void validateSelectExpr() throws WindowingException
192+
{
193+
if (selectSpec.getWindowFuncs() != null )
194+
{
195+
throw new WindowingException("NPath Result Expression cannot have Windowing Function expressions");
196+
}
197+
198+
for(ASTNode node : selectSpec.getExpressions())
199+
{
200+
TranslateUtils.validateNoLeadLagInValueBoundarySpec(node, "Lead/Lag not allowed in NPath Result Expression");
201+
}
202+
}
203+
}

‎windowing/src/main/java/com/sap/hadoop/windowing/query2/translate/TranslateUtils.java

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -693,18 +693,30 @@ public void visit(Object t, Object parent, int childIndex, Map labels)
693693

694694
public static void validateNoLeadLagInValueBoundarySpec(ASTNode node)
695695
throws WindowingException
696+
{
697+
validateNoLeadLagInValueBoundarySpec(node, "Lead/Lag not allowed in ValueBoundary Spec");
698+
}
699+
700+
public static void validateNoLeadLagInValueBoundarySpec(ASTNode node, String errMsg)
701+
throws WindowingException
696702
{
697703
TreeWizard tw = new TreeWizard(adaptor, Windowing2Parser.tokenNames);
698-
ValidateNoLeadLagInValueBoundarySpec visitor = new ValidateNoLeadLagInValueBoundarySpec();
704+
ValidateNoLeadLag visitor = new ValidateNoLeadLag(errMsg);
699705
tw.visit(node, Windowing2Parser.FUNCTION, visitor);
700706
visitor.checkValid();
701707
}
702708

703-
public static class ValidateNoLeadLagInValueBoundarySpec implements
709+
public static class ValidateNoLeadLag implements
704710
ContextVisitor
705711
{
712+
String errMsg;
706713
boolean throwError = false;
707714
ASTNode errorNode;
715+
716+
public ValidateNoLeadLag(String errMsg)
717+
{
718+
this.errMsg = errMsg;
719+
}
708720

709721
@SuppressWarnings("rawtypes")
710722
@Override
@@ -725,9 +737,7 @@ void checkValid() throws WindowingException
725737
{
726738
if (throwError)
727739
{
728-
throw new WindowingException(
729-
"Lead/Lag not allowed in ValueBoundary Spec"
730-
+ errorNode.toStringTree());
740+
throw new WindowingException(errMsg + errorNode.toStringTree());
731741
}
732742
}
733743
}

0 commit comments

Comments
 (0)