From 4497199f20c92897dee345dd5c96e0017416abd1 Mon Sep 17 00:00:00 2001 From: arnabp Date: Fri, 3 Sep 2021 00:07:38 +0200 Subject: [PATCH] [MINOR] Fix bugs in lineage tracing This patch fixes bugs in lineage tracing code and adds support for missing instructions. --- .../sysds/hops/recompile/LiteralReplacement.java | 10 ++++++++-- .../runtime/instructions/cp/DataGenCPInstruction.java | 10 ++++++++++ .../instructions/cp/FunctionCallCPInstruction.java | 5 +++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/recompile/LiteralReplacement.java b/src/main/java/org/apache/sysds/hops/recompile/LiteralReplacement.java index c36d53f092a..4740472ba6f 100644 --- a/src/main/java/org/apache/sysds/hops/recompile/LiteralReplacement.java +++ b/src/main/java/org/apache/sysds/hops/recompile/LiteralReplacement.java @@ -47,6 +47,7 @@ import org.apache.sysds.runtime.instructions.cp.ListObject; import org.apache.sysds.runtime.instructions.cp.ScalarObject; import org.apache.sysds.runtime.instructions.cp.ScalarObjectFactory; +import org.apache.sysds.runtime.lineage.LineageItem; import org.apache.sysds.runtime.matrix.data.MatrixBlock; import org.apache.sysds.utils.Statistics; @@ -81,7 +82,7 @@ protected static void rReplaceLiterals( Hop hop, ExecutionContext ec, boolean sc lit = (lit==null) ? replaceLiteralFullUnaryAggregateRightIndexing(c, vars) : lit; lit = (lit==null) ? replaceTReadMatrixFromList(c, ec) : lit; lit = (lit==null) ? replaceTReadMatrixFromListAppend(c, ec) : lit; - lit = (lit==null) ? replaceTReadMatrixLookupFromList(c, vars) : lit; + lit = (lit==null) ? replaceTReadMatrixLookupFromList(c, ec) : lit; lit = (lit==null) ? replaceTReadScalarLookupFromList(c, vars) : lit; } @@ -385,8 +386,9 @@ private static NaryOp replaceTReadMatrixFromListAppend( Hop c, ExecutionContext return ret; } - private static DataOp replaceTReadMatrixLookupFromList( Hop c, LocalVariableMap vars ) { + private static DataOp replaceTReadMatrixLookupFromList( Hop c, ExecutionContext ec) { //pattern: as.matrix(X[i:i]) or as.matrix(X['a','a']) with X being a list + LocalVariableMap vars = ec.getVariables(); DataOp ret = null; if( HopRewriteUtils.isUnary(c, OpOp1.CAST_AS_MATRIX) && c.getInput().get(0) instanceof IndexingOp ) { @@ -402,7 +404,11 @@ private static DataOp replaceTReadMatrixLookupFromList( Hop c, LocalVariableMap LiteralOp lit = (LiteralOp) ix.getInput().get(1); MatrixObject mo = (MatrixObject) (!lit.getValueType().isNumeric() ? list.slice(lit.getName()) : list.slice((int)lit.getLongValue()-1)); + LineageItem li = !lit.getValueType().isNumeric() ? + list.getLineageItem(lit.getName()) : list.getLineageItem((int)lit.getLongValue()-1); vars.put(varname, mo); + if (DMLScript.LINEAGE) + ec.getLineage().set(varname, li); ret = HopRewriteUtils.createTransientRead(varname, c); } } diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java index 9017ea82424..aa4f9ada6f2 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/DataGenCPInstruction.java @@ -528,6 +528,16 @@ public Pair getLineageItem(ExecutionContext ec) { tmpInstStr = replaceNonLiteral(tmpInstStr, seq_incr, 7, ec); break; } + case FRAMEINIT: { + tmpInstStr = InstructionUtils.replaceOperandName(tmpInstStr); + CPOperand frameInp = new CPOperand(frame_data, ValueType.STRING, DataType.SCALAR, true); + tmpInstStr = InstructionUtils.replaceOperand(tmpInstStr, 2, frameInp.getLineageLiteral()); + tmpInstStr = replaceNonLiteral(tmpInstStr, rows, 3, ec); + tmpInstStr = replaceNonLiteral(tmpInstStr, cols, 4, ec); + CPOperand schemaInp = new CPOperand(schema, ValueType.STRING, DataType.SCALAR, true); + tmpInstStr = !schema.equalsIgnoreCase("NULL") + ? InstructionUtils.replaceOperand(tmpInstStr, 5, schemaInp.getLineageLiteral()) : tmpInstStr; + } case TIME: // only opcode (time) is sufficient to compute from lineage. break; diff --git a/src/main/java/org/apache/sysds/runtime/instructions/cp/FunctionCallCPInstruction.java b/src/main/java/org/apache/sysds/runtime/instructions/cp/FunctionCallCPInstruction.java index df46b351cd1..cc0dfd9674e 100644 --- a/src/main/java/org/apache/sysds/runtime/instructions/cp/FunctionCallCPInstruction.java +++ b/src/main/java/org/apache/sysds/runtime/instructions/cp/FunctionCallCPInstruction.java @@ -313,8 +313,9 @@ private boolean reuseFunctionOutputs(LineageItem[] liInputs, FunctionProgramBloc return reuse; } - private static String getCacheFunctionName(String fname, FunctionProgramBlock fpb) { - return !fpb.hasThreadID() ? fname : + private String getCacheFunctionName(String fname, FunctionProgramBlock fpb) { + String tmpFname = !fpb.hasThreadID() ? fname : fname.substring(0, fname.lastIndexOf(Lop.CP_CHILD_THREAD+fpb.getThreadID())); + return DMLProgram.constructFunctionKey(_namespace, tmpFname); } }