diff --git a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java index 79c70efd04f..0f26fa1c749 100644 --- a/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java +++ b/src/main/java/org/apache/sysds/hops/rewrite/RewriteCompressedReblock.java @@ -127,8 +127,10 @@ private static void injectCompressionDirective(Hop hop, CompressConfig compress, } private static boolean satisfiesSizeConstraintsForCompression(Hop hop) { - return hop.getDim2() >= 1 && - ((hop.getDim1() >= 1000 && hop.getDim2() < 100) || hop.getDim1() / hop.getDim2() >= 1000); + if(hop.getDim2() >= 1) { + return (hop.getDim1() >= 1000 && hop.getDim2() < 100) || hop.getDim1() / hop.getDim2() >= 75; + } + return false; } private static boolean satisfiesCompressionCondition(Hop hop) { @@ -191,8 +193,11 @@ private static boolean satisfiesAutoCompressionCondition(Hop hop, DMLProgram pro } private static boolean satisfiesCostCompressionCondition(Hop hop, DMLProgram prog) { - return satisfiesAggressiveCompressionCondition(hop) && hop.dimsKnown(false) && - analyseProgram(hop, prog).isValidAggressiveCompression(); + boolean satisfies = true; + satisfies &= satisfiesAggressiveCompressionCondition(hop); + satisfies &= hop.dimsKnown(false); + satisfies &= analyseProgram(hop, prog).isValidAggressiveCompression(); + return satisfies; } @@ -210,7 +215,7 @@ private static class ProbeStatus { private int numberCompressedOpsExecuted = 0; private int numberDecompressedOpsExecuted = 0; private int inefficientSupportedOpsExecuted = 0; - private int superEfficientSuportedOpsExecuted = 0; + // private int superEfficientSupportedOpsExecuted = 0; private boolean foundStart = false; private boolean usedInLoop = false; @@ -363,7 +368,6 @@ private void handleFunctionOps(Hop current) { private void handleApplicableOps(Hop current) { // Valid with uncompressed outputs boolean compUCOut = false; - LOG.error(current); // // tsmm // compUCOut |= (current instanceof AggBinaryOp && current.getDim2() <= current.getBlocksize() && // ((AggBinaryOp) current).checkTransposeSelf() == MMTSJType.LEFT); @@ -403,7 +407,7 @@ private void handleApplicableOps(Hop current) { boolean metaOp = HopRewriteUtils.isUnary(current, OpOp1.NROW, OpOp1.NCOL); boolean ctableOp = HopRewriteUtils.isTernary(current, OpOp3.CTABLE); - if(ctableOp){ + if(ctableOp) { numberCompressedOpsExecuted += 4; compCOut = true; } @@ -431,7 +435,7 @@ private boolean isValidAggressiveCompression() { if(LOG.isDebugEnabled()) LOG.debug(this.toString()); return (inefficientSupportedOpsExecuted < numberCompressedOpsExecuted) && - (usedInLoop || numberCompressedOpsExecuted > 3) && numberDecompressedOpsExecuted < 1; + (usedInLoop || numberCompressedOpsExecuted > 3) && numberDecompressedOpsExecuted < 1; } @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java index f2958da3d96..5dc44d12e09 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlock.java @@ -205,35 +205,21 @@ public MatrixBlock decompress() { // preallocation sparse rows to avoid repeated reallocations MatrixBlock ret = new MatrixBlock(rlen, clen, false, -1); + ret.allocateDenseBlock(); - // (nonZeros == -1) ? - // .allocateBlock() : new MatrixBlock(rlen, clen, sparse, - // nonZeros).allocateBlock(); - - // if(ret.isInSparseFormat()) { - // int[] rnnz = new int[rlen]; - // // for(ColGroup grp : _colGroups) - // // grp.countNonZerosPerRow(rnnz, 0, rlen); - // ret.allocateSparseRowsBlock(); - // SparseBlock rows = ret.getSparseBlock(); - // for(int i = 0; i < rlen; i++) - // rows.allocate(i, rnnz[i]); - // } + // todo Add sparse decompress. - // core decompression (append if sparse) for(AColGroup grp : _colGroups) grp.decompressToBlockUnSafe(ret, 0, rlen, 0, grp.getValues()); - // post-processing (for append in decompress) - if(ret.getNonZeros() == -1 || nonZeros == -1) { - ret.recomputeNonZeros(); - } - else { - ret.setNonZeros(nonZeros); - } if(ret.isInSparseFormat()) ret.sortSparseRows(); + if(nonZeros == -1) + ret.setNonZeros(this.recomputeNonZeros()); + else + ret.setNonZeros(nonZeros); + if(DMLScript.STATISTICS || LOG.isDebugEnabled()) { double t = time.stop(); LOG.debug("decompressed block w/ k=" + 1 + " in " + t + "ms."); @@ -256,9 +242,12 @@ public MatrixBlock decompress(int k) { Timing time = new Timing(true); MatrixBlock ret = new MatrixBlock(rlen, clen, false, -1).allocateBlock(); + ret.allocateDenseBlock(); + if(nonZeros == -1) + ret.setNonZeros(this.recomputeNonZeros()); + else + ret.setNonZeros(nonZeros); - nonZeros = 0; - boolean overlapping = isOverlapping(); try { ExecutorService pool = CommonThreadPool.get(k); int rlen = getNumRows(); @@ -268,30 +257,24 @@ public MatrixBlock decompress(int k) { ArrayList tasks = new ArrayList<>(); for(int i = 0; i < k & i * blklen < getNumRows(); i++) tasks.add( - new DecompressTask(_colGroups, ret, i * blklen, Math.min((i + 1) * blklen, rlen), overlapping)); + new DecompressTask(_colGroups, ret, i * blklen, Math.min((i + 1) * blklen, rlen), overlappingColGroups)); List> rtasks = pool.invokeAll(tasks); pool.shutdown(); for(Future rt : rtasks) - nonZeros += rt.get(); // error handling + rt.get(); // error handling } catch(InterruptedException | ExecutionException ex) { LOG.error("Parallel decompression failed defaulting to non parallel implementation " + ex.getMessage()); - nonZeros = -1; ex.printStackTrace(); return decompress(); } - if(overlapping) { - ret.recomputeNonZeros(); - } - else { - ret.setNonZeros(nonZeros); - } if(DMLScript.STATISTICS || LOG.isDebugEnabled()) { double t = time.stop(); LOG.debug("decompressed block w/ k=" + k + " in " + time.stop() + "ms."); DMLCompressionStatistics.addDecompressTime(t, k); } + return ret; } @@ -299,6 +282,22 @@ public CompressedMatrixBlock squash(int k) { return CLALibSquash.squash(this, k); } + @Override + public long recomputeNonZeros() { + if(overlappingColGroups) { + nonZeros = clen * rlen; + } + else { + long nnz = 0; + for(AColGroup g : _colGroups) { + nnz += g.getNumberNonZeros(); + } + nonZeros = nnz; + } + return nonZeros; + + } + /** * Obtain an upper bound on the memory used to store the compressed block. * @@ -497,6 +496,7 @@ else if(ctype == ChainType.XtwXv) CLALibLeftMultBy.leftMultByMatrixTransposed(this, tmp, out, k); out = LibMatrixReorg.transposeInPlace(out, k); + out.recomputeNonZeros(); return out; } @@ -811,7 +811,7 @@ else if(cl == 0 && cu == getNumColumns() - 1) { tmp = new MatrixBlock(ru + 1 - rl, getNumColumns(), false).allocateDenseBlock(); for(AColGroup g : getColGroups()) g.decompressToBlock(tmp, rl, ru + 1, 0); - + tmp.recomputeNonZeros(); return tmp; } else { @@ -825,6 +825,7 @@ else if(cl == 0 && cu == getNumColumns() - 1) { // this is fine. tmp = tmp.slice(rl, ru, 0, tmp.getNumColumns() - 1, ret); } + tmp.recomputeNonZeros(); ret = tmp; return tmp; } @@ -1234,7 +1235,7 @@ public MatrixBlock randOperationsInPlace(RandomMatrixGenerator rgen, Well1024a b public MatrixBlock randOperationsInPlace(RandomMatrixGenerator rgen, Well1024a bigrand, long bSeed, int k) { throw new DMLRuntimeException("CompressedMatrixBlock: randOperationsInPlace not supported."); } - + @Override public MatrixBlock seqOperationsInPlace(double from, double to, double incr) { // output should always be uncompressed diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java index 0fd05e4fd1c..0cc75af6487 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressedMatrixBlockFactory.java @@ -127,6 +127,7 @@ private Pair compressMatrix() { if(res == null) return abortCompression(); + res.recomputeNonZeros(); return new ImmutablePair<>(res, _stats); } @@ -190,12 +191,15 @@ private void cleanupPhase() { mb.cleanupBlock(true, true); _stats.size = res.estimateCompressedSizeInMemory(); - _stats.originalSize = mb.estimateSizeInMemory(); + _stats.originalSize = original.estimateSizeInMemory(); + _stats.denseSize = MatrixBlock.estimateSizeInMemory(original.getNumRows(), original.getNumColumns(), 1.0); _stats.ratio = _stats.originalSize / (double) _stats.size; if(_stats.ratio < 1) { - LOG.info("--compressed size: " + _stats.size); - LOG.info("--compression ratio: " + _stats.ratio); + LOG.info("--dense size: " + _stats.denseSize); + LOG.info("--original size: " + _stats.originalSize); + LOG.info("--compressed size: " + _stats.size); + LOG.info("--compression ratio: " + _stats.ratio ); LOG.info("Abort block compression because compression ratio is less than 1."); res = null; return; @@ -239,8 +243,10 @@ private void logPhase() { LOG.debug("--compression phase " + phase + " Cleanup : " + _stats.getLastTimePhase()); LOG.debug("--col groups types " + _stats.getGroupsTypesString()); LOG.debug("--col groups sizes " + _stats.getGroupsSizesString()); - LOG.debug("--compressed size: " + _stats.size); - LOG.debug("--compression ratio: " + _stats.ratio); + LOG.debug("--dense size: " + _stats.denseSize); + LOG.debug("--original size: " + _stats.originalSize); + LOG.debug("--compressed size: " + _stats.size); + LOG.debug("--compression ratio: " + _stats.ratio ); int[] lengths = new int[res.getColGroups().size()]; int i = 0; for(AColGroup colGroup : res.getColGroups()) { diff --git a/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java b/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java index 83fadd28c14..41ac2a02477 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java +++ b/src/main/java/org/apache/sysds/runtime/compress/CompressionStatistics.java @@ -31,6 +31,7 @@ public class CompressionStatistics { private double lastPhase; public double ratio; public long originalSize; + public long denseSize; public long estimatedSizeColGroups; public long estimatedSizeCols; public long size; diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java index 808d43b61b9..33e9d112df1 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/AColGroup.java @@ -646,6 +646,8 @@ public boolean isDense() { public abstract boolean containsValue(double pattern); + public abstract long getNumberNonZeros(); + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java index 8422a3f00ae..10b069e00ef 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ADictionary.java @@ -250,4 +250,6 @@ public static ADictionary read(DataInput in, boolean lossy) throws IOException { public abstract ADictionary reExpandColumns(int max); public abstract boolean containsValue(double pattern); + + public abstract long getNumberNonZeros(int[] counts, int nCol); } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java index a70d25034f5..7cda071aeb9 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupEmpty.java @@ -38,6 +38,7 @@ protected ColGroupEmpty() { super(); } + /** * Constructs an Constant Colum Group, that contains only one tuple, with the given value. * @@ -49,6 +50,14 @@ public ColGroupEmpty(int[] colIndices, int numRows) { _zeros = true; } + public static ColGroupEmpty generate(int nCol, int nRow){ + int[] cols = new int[nCol]; + for(int i =0; i < nCol; i++){ + cols[i] =i; + } + return new ColGroupEmpty(cols,nRow); + } + @Override public int[] getCounts(int[] out) { // nothing diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java index 824ff8c140c..1722ffdaf12 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupUncompressed.java @@ -543,4 +543,9 @@ public AColGroup copy() { public boolean containsValue(double pattern){ return _data.containsValue(pattern); } + + @Override + public long getNumberNonZeros(){ + return _data.getNonZeros(); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java index d0a6ee96824..085dd1d807f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/ColGroupValue.java @@ -1030,4 +1030,10 @@ public void leftMultBySelfDiagonalColGroup(double[] result, int numColumns) { public boolean containsValue(double pattern){ return _dict.containsValue(pattern); } + + @Override + public long getNumberNonZeros(){ + int[] counts = getCounts(); + return _dict.getNumberNonZeros(counts, _colIndexes.length); + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java index 658decb66bd..34fea3d7924 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/Dictionary.java @@ -387,4 +387,20 @@ public boolean containsValue(double pattern) { return false; } + + @Override + public long getNumberNonZeros(int[] counts, int nCol){ + long nnz = 0; + final int nRow = _values.length / nCol; + for(int i = 0; i < nRow; i++){ + long rowCount = 0; + final int off = i * nCol; + for(int j = off; j < off + nCol; j++){ + if(_values[j] != 0) + rowCount ++; + } + nnz += rowCount * counts[i]; + } + return nnz; + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java b/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java index 05b1817586b..8986d71ec63 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java +++ b/src/main/java/org/apache/sysds/runtime/compress/colgroup/QDictionary.java @@ -452,4 +452,20 @@ public boolean containsValue(double pattern){ return false; throw new NotImplementedException("Not contains value on Q Dictionary"); } + + @Override + public long getNumberNonZeros(int[] counts, int nCol){ + long nnz = 0; + final int nRow = _values.length / nCol; + for(int i = 0; i < nRow; i++){ + long rowCount = 0; + final int off = i * nCol; + for(int j = off; j < off + nCol; j++){ + if(_values[j] != 0) + rowCount ++; + } + nnz += rowCount * counts[i]; + } + return nnz; + } } diff --git a/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java b/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java index ebe556625ab..7ae813fc53b 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java +++ b/src/main/java/org/apache/sysds/runtime/compress/estim/CompressedSizeInfoColGroup.java @@ -122,7 +122,7 @@ private static Long getCompressionSize(CompressionType ct, EstimationFactors fac case UNCOMPRESSED: return ColGroupSizes.estimateInMemorySizeUncompressed(fact.numRows, fact.numCols, - ((double) fact.numVals / (fact.numRows * fact.numCols))); + ((double) fact.numOffs / (fact.numRows * fact.numCols))); case SDC: if(fact.numOffs == 1) return ColGroupSizes.estimateInMemorySizeSDCSingle(fact.numCols, diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java index c6bf61d8ac8..bb130b6f509 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibAppend.java @@ -20,6 +20,7 @@ package org.apache.sysds.runtime.compress.lib; import java.util.ArrayList; +import java.util.List; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.logging.Log; @@ -28,6 +29,7 @@ import org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory; import org.apache.sysds.runtime.compress.CompressionStatistics; import org.apache.sysds.runtime.compress.colgroup.AColGroup; +import org.apache.sysds.runtime.compress.colgroup.ColGroupEmpty; import org.apache.sysds.runtime.matrix.data.MatrixBlock; public class CLALibAppend { @@ -35,27 +37,25 @@ public class CLALibAppend { private static final Log LOG = LogFactory.getLog(CLALibAppend.class.getName()); public static MatrixBlock append(MatrixBlock left, MatrixBlock right) { - - // if(left.isEmpty()) - // return right; - // else if(right.isEmpty()) - // return left; + + if(left.isEmpty() && right instanceof CompressedMatrixBlock) + return appendLeftEmpty(left, (CompressedMatrixBlock) right); + else if(right.isEmpty() && left instanceof CompressedMatrixBlock) + return appendRightEmpty((CompressedMatrixBlock)left, right); + final int m = left.getNumRows(); final int n = left.getNumColumns() + right.getNumColumns(); - long nnz = left.getNonZeros() + right.getNonZeros(); - if(left.getNonZeros() < 0 || right.getNonZeros() < 0) - nnz = -1; // try to compress both sides (if not already compressed). - if(!(left instanceof CompressedMatrixBlock) && m > 1000){ - LOG.info("Compressing left for append operation"); + if(!(left instanceof CompressedMatrixBlock) && m > 1000) { + LOG.warn("Compressing left for append operation"); Pair x = CompressedMatrixBlockFactory.compress(left); if(x.getRight().ratio > 3.0) left = x.getLeft(); - + } - if(!(right instanceof CompressedMatrixBlock) && m > 1000){ - LOG.info("Compressing right for append operation"); + if(!(right instanceof CompressedMatrixBlock) && m > 1000) { + LOG.warn("Compressing right for append operation"); Pair x = CompressedMatrixBlockFactory.compress(right); if(x.getRight().ratio > 3.0) right = x.getLeft(); @@ -71,18 +71,53 @@ public static MatrixBlock append(MatrixBlock left, MatrixBlock right) { // init result matrix CompressedMatrixBlock ret = new CompressedMatrixBlock(m, n); + ret = appendColGroups(ret, leftC.getColGroups(), rightC.getColGroups(), leftC.getNumColumns()); + return ret; + } + + private static MatrixBlock appendRightEmpty(CompressedMatrixBlock left, MatrixBlock right) { + + final int m = left.getNumRows(); + final int n = left.getNumColumns() + right.getNumColumns(); + CompressedMatrixBlock ret = new CompressedMatrixBlock(m,n); + + List newGroup = new ArrayList<>(1); + newGroup.add(ColGroupEmpty.generate(right.getNumColumns(), right.getNumRows())); + ret = appendColGroups(ret, left.getColGroups(), newGroup, left.getNumColumns()); + + return ret; + } + + private static MatrixBlock appendLeftEmpty(MatrixBlock left, CompressedMatrixBlock right) { + final int m = left.getNumRows(); + final int n = left.getNumColumns() + right.getNumColumns(); + CompressedMatrixBlock ret = new CompressedMatrixBlock(m,n); + + List newGroup = new ArrayList<>(1); + newGroup.add(ColGroupEmpty.generate(left.getNumColumns(), left.getNumRows())); + ret = appendColGroups(ret, newGroup, right.getColGroups(), left.getNumColumns()); + + return ret; + } + + private static CompressedMatrixBlock appendColGroups(CompressedMatrixBlock ret, List left, + List right, int leftNumCols) { + // shallow copy of lhs column groups - ret.allocateColGroupList(new ArrayList(leftC.getColGroups().size() + rightC.getColGroups().size())); + ret.allocateColGroupList(new ArrayList(left.size() + right.size())); - for(AColGroup group : leftC.getColGroups()){ + long nnz = 0; + for(AColGroup group : left) { AColGroup tmp = group.copy(); ret.getColGroups().add(tmp); + nnz += group.getNumberNonZeros(); } - for(AColGroup group : rightC.getColGroups()) { + for(AColGroup group : right) { AColGroup tmp = group.copy(); - tmp.shiftColIndices(left.getNumColumns()); + tmp.shiftColIndices(leftNumCols); ret.getColGroups().add(tmp); + nnz += group.getNumberNonZeros(); } // meta data maintenance diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java index 2eb68887552..f7b288d5593 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibCompAgg.java @@ -56,7 +56,7 @@ public class CLALibCompAgg { // private static final Log LOG = LogFactory.getLog(LibCompAgg.class.getName()); // private static final long MIN_PAR_AGG_THRESHOLD = 8 * 1024 * 1024; - private static final long MIN_PAR_AGG_THRESHOLD = 8; + private static final long MIN_PAR_AGG_THRESHOLD = 8 * 1024 ; private static ThreadLocal memPool = new ThreadLocal() { @Override diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java index f6390c68b57..3ef03f31e6a 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibLeftMultBy.java @@ -59,7 +59,9 @@ protected double[] initialValue() { public static MatrixBlock leftMultByMatrixTransposed(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k){ MatrixBlock transposed = new MatrixBlock(m2.getNumColumns(), m2.getNumRows(), false); LibMatrixReorg.transpose(m2, transposed); - return leftMultByMatrix(m1, transposed, ret, k ); + ret = leftMultByMatrix(m1, transposed, ret, k ); + ret.recomputeNonZeros(); + return ret; // return LibMatrixReorg.transpose(ret, new MatrixBlock(ret.getNumColumns(), ret.getNumRows(), false)); } @@ -75,8 +77,10 @@ public static MatrixBlock leftMultByMatrixTransposed(CompressedMatrixBlock m1, C public static MatrixBlock leftMultByMatrix(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k) { prepareReturnMatrix(m1, m2, ret, false); - return leftMultByMatrix(m1 + ret = leftMultByMatrix(m1 .getColGroups(), m2, ret, false, m1.getNumColumns(), m1.isOverlapping(), k, m1.getMaxNumValues()); + ret.recomputeNonZeros(); + return ret; } private static MatrixBlock leftMultByMatrix(List groups, MatrixBlock that, MatrixBlock ret, @@ -172,48 +176,6 @@ private static void leftMultBySelfDiagonalColGroup(List groups, Matri } } - // public static MatrixBlock leftMultByVectorTranspose(List colGroups, MatrixBlock vector, - // MatrixBlock result, boolean doTranspose, int k, Pair v, boolean overlap) { - - // // transpose vector if required - // MatrixBlock rowVector = vector; - // if(doTranspose) { - // rowVector = new MatrixBlock(1, vector.getNumRows(), false); - // LibMatrixReorg.transpose(vector, rowVector); - // } - - // result.reset(); - // result.allocateDenseBlock(); - - // // multi-threaded execution - // try { - // // compute uncompressed column group in parallel - // // ColGroupUncompressed uc = getUncompressedColGroup(); - // // if(uc != null) - // // uc.leftMultByRowVector(rowVector, result, k); - - // // compute remaining compressed column groups in parallel - // ExecutorService pool = CommonThreadPool.get(Math.min(colGroups.size(), k)); - // ArrayList tasks = new ArrayList<>(); - - // tasks.add(new LeftMatrixVectorMultTask(colGroups, rowVector, result, v)); - - // List> ret = pool.invokeAll(tasks); - // pool.shutdown(); - // for(Future tmp : ret) - // tmp.get(); - - // } - // catch(InterruptedException | ExecutionException e) { - // throw new DMLRuntimeException(e); - // } - - // // post-processing - // result.recomputeNonZeros(); - - // return result; - // } - private static MatrixBlock leftMultByCompressedTransposedMatrix(List colGroups, CompressedMatrixBlock that, MatrixBlock ret, int k, int numColumns, Pair v, boolean overlapping) { diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java index 97cf12845b5..2b4e3188407 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibReExpand.java @@ -68,8 +68,8 @@ private static CompressedMatrixBlock reExpandRows(CompressedMatrixBlock in, Comp ret.allocateColGroupList(newColGroups); ret.setOverlapping(true); - ret.setNonZeros(-1); + ret.recomputeNonZeros(); return ret; } diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java index 8ce20f260e1..ec2849751de 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibRightMultBy.java @@ -42,7 +42,9 @@ public class CLALibRightMultBy { private static final Log LOG = LogFactory.getLog(CLALibRightMultBy.class.getName()); public static MatrixBlock rightMultByMatrix(CompressedMatrixBlock m1, MatrixBlock m2, MatrixBlock ret, int k, boolean allowOverlap){ - return rightMultByMatrix(m1.getColGroups(), m2, ret, k, m1.getMaxNumValues(), allowOverlap); + ret = rightMultByMatrix(m1.getColGroups(), m2, ret, k, m1.getMaxNumValues(), allowOverlap); + ret.recomputeNonZeros(); + return ret; } private static MatrixBlock rightMultByMatrix(List colGroups, MatrixBlock that, MatrixBlock ret, int k, diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java index 97a7f6769e8..1c0a9809a0f 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibScalar.java @@ -57,7 +57,9 @@ public class CLALibScalar { public static MatrixBlock scalarOperations(ScalarOperator sop, CompressedMatrixBlock m1, MatrixValue result) { // Special case handling of overlapping relational operations if(CLALibRelationalOp.isValidForRelationalOperation(sop, m1)) { - return CLALibRelationalOp.overlappingRelativeRelationalOperation(sop, m1); + MatrixBlock ret = CLALibRelationalOp.overlappingRelativeRelationalOperation(sop, m1); + ret.recomputeNonZeros(); + return ret; } if(isInvalidForCompressedOutput(m1, sop)) { @@ -96,7 +98,7 @@ public static MatrixBlock scalarOperations(ScalarOperator sop, CompressedMatrixB ret.setOverlapping(m1.isOverlapping()); } - ret.setNonZeros(-1); + ret.recomputeNonZeros(); return ret; diff --git a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java index b4438b27dd5..63177c5be25 100644 --- a/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java +++ b/src/main/java/org/apache/sysds/runtime/compress/lib/CLALibSquash.java @@ -59,7 +59,7 @@ public static CompressedMatrixBlock squash(CompressedMatrixBlock m, int k) { ret.allocateColGroupList(retCg); ret.setOverlapping(false); - ret.setNonZeros(-1); + ret.recomputeNonZeros(); if(ret.isOverlapping()) throw new DMLCompressionException("Squash should output compressed nonOverlapping matrix"); diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java index 06de8f70fea..cf6ab3fc3d7 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/CacheableData.java @@ -1040,7 +1040,7 @@ protected abstract T reconstructByLineage(LineageItem li) protected void writeMetaData (String filePathAndName, String outputFormat, FileFormatProperties formatProperties) throws IOException - { + { MetaDataFormat iimd = (MetaDataFormat) _metaData; if (iimd == null) diff --git a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java index d0bce6eb723..e55509ba7d7 100644 --- a/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysds/runtime/controlprogram/caching/MatrixObject.java @@ -35,6 +35,7 @@ import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.lops.Lop; import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat; import org.apache.sysds.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysds.runtime.controlprogram.federated.FederatedRange; @@ -583,13 +584,15 @@ protected void writeBlobToHDFS(String fname, String ofmt, int rep, FileFormatPro begin = System.currentTimeMillis(); } - MetaDataFormat iimd = (MetaDataFormat) _metaData; - if(this.isFederated() && FileFormat.safeValueOf(ofmt) == FileFormat.FEDERATED){ ReaderWriterFederated.write(fname,this._fedMapping); } else if (_data != null) { + if(_data instanceof CompressedMatrixBlock) + _data = CompressedMatrixBlock.getUncompressed(_data); + + MetaDataFormat iimd = (MetaDataFormat) _metaData; // Get the dimension information from the metadata stored within MatrixObject DataCharacteristics mc = iimd.getDataCharacteristics(); // Write the matrix to HDFS in requested format diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java b/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java index b7d09e3c96f..0d414fa3c4d 100644 --- a/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java +++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlock.java @@ -25,6 +25,7 @@ import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.instructions.cp.KahanObject; import org.apache.sysds.runtime.util.UtilFunctions; +import org.apache.sysds.utils.MemoryEstimates; /** * This DenseBlock is an abstraction for different dense, row-major @@ -674,4 +675,13 @@ private static int[] createDimOffsets(int[] dims) { } return ret; } + + public static long estimateSizeDenseInMemory(int nRows, int nCols){ + long size = 16; // object + size += 4; // int + size += 4; // padding + size += MemoryEstimates.intArrayCost(1); // odims typically 1 + size += 8; // pointer to reuse that is typically null; + return size; + } } diff --git a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java index 3d38f552e19..795bee19516 100644 --- a/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java +++ b/src/main/java/org/apache/sysds/runtime/data/DenseBlockFP64.java @@ -21,6 +21,7 @@ package org.apache.sysds.runtime.data; import org.apache.sysds.runtime.util.UtilFunctions; +import org.apache.sysds.utils.MemoryEstimates; import java.util.Arrays; @@ -192,4 +193,10 @@ public String getString(int[] ix) { public long getLong(int[] ix) { return UtilFunctions.toLong(_data[pos(ix)]); } + + public static long estimateSizeDenseInMemory(int nRows, int nCols){ + long size = DenseBlock.estimateSizeDenseInMemory(nRows, nCols);// pointer to reuse that is typically null; + size += MemoryEstimates.doubleArrayCost(nRows * nCols); + return size; + } } diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java index d876946be68..4375cad206a 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlock.java @@ -22,6 +22,8 @@ import java.io.Serializable; import java.util.Iterator; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.sysds.runtime.matrix.data.IJV; /** @@ -37,6 +39,9 @@ */ public abstract class SparseBlock implements Serializable { + + protected static final Log LOG = LogFactory.getLog(SparseBlock.class.getName()); + private static final long serialVersionUID = -5008747088111141395L; //internal configuration parameters for all sparse blocks diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java index 37199fa2219..ce8e7077b73 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCOO.java @@ -25,6 +25,7 @@ import org.apache.sysds.runtime.matrix.data.IJV; import org.apache.sysds.runtime.util.SortUtils; import org.apache.sysds.runtime.util.UtilFunctions; +import org.apache.sysds.utils.MemoryEstimates; /** * SparseBlock implementation that realizes a traditional 'coordinate matrix' @@ -149,9 +150,9 @@ public static long estimateMemory(long nrows, long ncols, double sparsity) { //32B overhead per array, int/int/double arr in nnz double size = 16 + 8; //object + 2 int fields - size += 24 + lnnz * 4d; //rindexes array (row indexes) - size += 24 + lnnz * 4d; //cindexes array (column indexes) - size += 24 + lnnz * 8d; //values array (non-zero values) + size += MemoryEstimates.intArrayCost((int)lnnz); //rindexes array (row indexes) + size += MemoryEstimates.intArrayCost((int) lnnz); //cindexes array (column indexes) + size += MemoryEstimates.doubleArrayCost((int) lnnz); //values array (non-zero values) //robustness for long overflows return (long) Math.min(size, Long.MAX_VALUE); diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java index 621a92a76aa..6cf474fa224 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockCSR.java @@ -25,6 +25,7 @@ import org.apache.sysds.runtime.util.SortUtils; import org.apache.sysds.runtime.util.UtilFunctions; +import org.apache.sysds.utils.MemoryEstimates; /** * SparseBlock implementation that realizes a traditional 'compressed sparse row' @@ -267,10 +268,10 @@ public static long estimateMemory(long nrows, long ncols, double sparsity) { double lnnz = Math.max(INIT_CAPACITY, Math.ceil(sparsity*nrows*ncols)); //32B overhead per array, int arr in nrows, int/double arr in nnz - double size = 16 + 4; //object + int field - size += 24 + (nrows+1) * 4d; //ptr array (row pointers) - size += 24 + lnnz * 4d; //indexes array (column indexes) - size += 24 + lnnz * 8d; //values array (non-zero values) + double size = 16 + 4 + 4; //object + int field + padding + size += MemoryEstimates.intArrayCost((int)nrows+1); //ptr array (row pointers) + size += MemoryEstimates.intArrayCost((int) lnnz); //indexes array (column indexes) + size += MemoryEstimates.doubleArrayCost((int) lnnz);//values array (non-zero values) //robustness for long overflows return (long) Math.min(size, Long.MAX_VALUE); diff --git a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java index 707298ebee1..fda83bf09ee 100644 --- a/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java +++ b/src/main/java/org/apache/sysds/runtime/data/SparseBlockMCSR.java @@ -19,6 +19,8 @@ package org.apache.sysds.runtime.data; +import org.apache.sysds.utils.MemoryEstimates; + /** * SparseBlock implementation that realizes a 'modified compressed sparse row' * representation, where each compressed row is stored as a separate SparseRow @@ -98,8 +100,8 @@ public SparseBlockMCSR(int rlen, int clen) { * @return memory estimate */ public static long estimateMemory(long nrows, long ncols, double sparsity) { - double cnnz = Math.max(SparseRowVector.initialCapacity, Math.ceil(sparsity*ncols)); - double rlen = Math.min(nrows, Math.ceil(sparsity*nrows*ncols)); + int cnnz = Math.max(SparseRowVector.initialCapacity, (int) Math.ceil(sparsity*ncols)); + double rlen = Math.min(nrows, Math.ceil(sparsity*nrows*ncols)); //Each sparse row has a fixed overhead of 16B (object) + 12B (3 ints), //24B (int array), 24B (double array), i.e., in total 76B @@ -107,8 +109,13 @@ public static long estimateMemory(long nrows, long ncols, double sparsity) { //Overheads for arrays, objects, and references refer to 64bit JVMs //If nnz < rows we have guaranteed also empty rows. double size = 16; //object - size += 24 + nrows * 8d; //references - size += rlen * (76 + cnnz * 12); //sparse rows + size += MemoryEstimates.objectArrayCost((int)rlen); //references + long sparseRowSize = 16; // object + sparseRowSize += MemoryEstimates.intArrayCost(cnnz); + sparseRowSize += MemoryEstimates.doubleArrayCost(cnnz); + sparseRowSize += 4*3; // integers. + sparseRowSize += 4; // padding to nearest 8 byte. + size += rlen * sparseRowSize; //sparse rows // robustness for long overflows return (long) Math.min(size, Long.MAX_VALUE); diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java index a8fce54a507..2f1a51f8d13 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/LibMatrixDatagen.java @@ -33,6 +33,7 @@ import org.apache.commons.math3.random.Well1024a; import org.apache.sysds.hops.DataGenOp; import org.apache.sysds.runtime.DMLRuntimeException; +import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.controlprogram.parfor.util.IDSequence; import org.apache.sysds.runtime.data.DenseBlock; import org.apache.sysds.runtime.data.SparseBlock; @@ -186,6 +187,9 @@ public static void generateRandomMatrix( MatrixBlock out, RandomMatrixGenerator int blen = rgen._blocksize; double sparsity = rgen._sparsity; + if(out instanceof CompressedMatrixBlock) + throw new DMLRuntimeException("Invalid to use compressed matrix block as output"); + // sanity check valid dimensions and sparsity checkMatrixDimensionsAndSparsity(rows, cols, sparsity); @@ -258,6 +262,9 @@ public static void generateRandomMatrix( MatrixBlock out, RandomMatrixGenerator int blen = rgen._blocksize; double sparsity = rgen._sparsity; + if(out instanceof CompressedMatrixBlock) + throw new DMLRuntimeException("Invalid to use compressed matrix block as output"); + //sanity check valid dimensions and sparsity checkMatrixDimensionsAndSparsity(rows, cols, sparsity); @@ -496,13 +503,15 @@ private static void genRandomNumbers(boolean invokedFromCP, int rl, int ru, int // are always selected uniformly at random. nnzPRNG.setSeed(seed); - // block-level sparsity, which may differ from overall sparsity in the matrix. - // (e.g., border blocks may fall under skinny matrix turn point, in CP this would be - // irrelevant but we need to ensure consistency with MR) boolean localSparse = MatrixBlock.evalSparseFormatInMemory( blockrows, blockcols, (long)(sparsity*blockrows*blockcols)); - if ( localSparse ) { + if ( localSparse) { + SparseBlock c = out.sparseBlock; + if(c == null){ + out.allocateSparseRowsBlock(); + c = out.sparseBlock; + } // Prob [k-1 zeros before a nonzero] = Prob [k-1 < log(uniform)/log(1-p) < k] = p*(1-p)^(k-1), where p=sparsity double log1mp = Math.log(1-sparsity); int idx = 0; // takes values in range [1, blen*blen] (both ends including) diff --git a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java index 96dd709f678..ff8cf31f240 100644 --- a/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysds/runtime/matrix/data/MatrixBlock.java @@ -52,6 +52,7 @@ import org.apache.sysds.runtime.controlprogram.caching.LazyWriteBuffer; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject.UpdateType; import org.apache.sysds.runtime.data.DenseBlock; +import org.apache.sysds.runtime.data.DenseBlockFP64; import org.apache.sysds.runtime.data.DenseBlockFactory; import org.apache.sysds.runtime.data.SparseBlock; import org.apache.sysds.runtime.data.SparseBlockCOO; @@ -2432,24 +2433,24 @@ public static long estimateSizeInMemory(long nrows, long ncols, double sparsity) { //determine sparse/dense representation boolean sparse = evalSparseFormatInMemory(nrows, ncols, (long)(sparsity*nrows*ncols)); - + + // basic variables and references sizes + long size = 16; // header + size += 12; // ints + size += 1; // boolean + size += 3; // padding + size += 8 * 2; // object references + //estimate memory consumption for sparse/dense if( sparse ) - return estimateSizeSparseInMemory(nrows, ncols, sparsity); + return size + estimateSizeSparseInMemory(nrows, ncols, sparsity); else - return estimateSizeDenseInMemory(nrows, ncols); + return size + estimateSizeDenseInMemory(nrows, ncols); } public static long estimateSizeDenseInMemory(long nrows, long ncols) { - // basic variables and references sizes - double size = 44; - - // core dense matrix block (double array) - size += 8d * nrows * ncols; - - // robustness for long overflows - return (long) Math.min(size, Long.MAX_VALUE); + return (long) Math.min(DenseBlockFP64.estimateSizeDenseInMemory((int)nrows, (int)ncols), Long.MAX_VALUE); } public static long estimateSizeSparseInMemory(long nrows, long ncols, double sparsity) { @@ -2458,15 +2459,9 @@ public static long estimateSizeSparseInMemory(long nrows, long ncols, double spa public static long estimateSizeSparseInMemory(long nrows, long ncols, double sparsity, SparseBlock.Type stype) { - // basic variables and references sizes - double size = 44; - // delegate memory estimate to individual sparse blocks - size += SparseBlockFactory.estimateSizeSparseInMemory( - stype, nrows, ncols, sparsity); - - // robustness for long overflows - return (long) Math.min(size, Long.MAX_VALUE); + return Math.min(SparseBlockFactory.estimateSizeSparseInMemory( + stype, nrows, ncols, sparsity),Long.MAX_VALUE); } public long estimateSizeOnDisk() diff --git a/src/main/java/org/apache/sysds/utils/MemoryEstimates.java b/src/main/java/org/apache/sysds/utils/MemoryEstimates.java index c22e47e2b97..473332fdf0d 100644 --- a/src/main/java/org/apache/sysds/utils/MemoryEstimates.java +++ b/src/main/java/org/apache/sysds/utils/MemoryEstimates.java @@ -21,12 +21,20 @@ /** * Memory Estimates is a helper class containing static classes that estimate the memory requirements of different types - * of objects in java. All estimates are worst case JVM x86-64bit uncompressed object pointers. This in practice means - * that the objects are most commonly smaller, for instance the object references are often time (at low memory pressure - * 4 bits) + * of objects in java. All estimates are worst case JVM x86-64bit uncompressed object pointers. + * + * This in practice means that the objects are most commonly smaller, for instance the object references are often time. + * + * If the memory pressure is low (there is a low number of allocated objects) then object pointers are 4 bits. */ public class MemoryEstimates { + /** + * Get the worst case memory usage of an java.util.BitSet java object. + * + * @param length The length of the array. + * @return The memory estimate in bytes + */ public static long bitSetCost(int length) { long size = 0; size += 8; // object reference @@ -37,6 +45,12 @@ public static long bitSetCost(int length) { return size; } + /** + * Get the worst case memory usage of an array of bytes. + * + * @param length The length of the array. + * @return The memory estimate in bytes + */ public static long byteArrayCost(int length) { long size = 0; size += 8; // Byte array Reference @@ -54,6 +68,12 @@ public static long byteArrayCost(int length) { return size; } + /** + * Get the worst case memory usage of an array of chars. + * + * @param length The length of the array. + * @return The memory estimate in bytes + */ public static long charArrayCost(int length) { long size = 0; size += 8; // char array Reference @@ -71,6 +91,12 @@ public static long charArrayCost(int length) { return size; } + /** + * Get the worst case memory usage of an array of integers. + * + * @param length The length of the array. + * @return The memory estimate in bytes + */ public static long intArrayCost(int length) { long size = 0; size += 8; // _ptr int[] reference @@ -87,6 +113,12 @@ public static long intArrayCost(int length) { return size; } + /** + * Get the worst case memory usage of an array of doubles. + * + * @param length The length of the array. + * @return The memory estimate in bytes + */ public static long doubleArrayCost(int length) { long size = 0; size += 8; // _values double array reference @@ -96,6 +128,27 @@ public static long doubleArrayCost(int length) { return size; } + /** + * Get the worst case memory usage for an array of objects. + * + * @param length The length of the array. + * @return The memory estimate in bytes + */ + public static long objectArrayCost(int length) { + long size = 0; + size += 8; // reference to array + size += 20; // header + size += 4; // padding before first reference + size += 8 * length; // references to all objects. + return size; + } + + /** + * Get the worst case memory usage for an array of longs + * + * @param length The length of the array. + * @return The memory estimate in bytes + */ public static long longArrayCost(int length) { return doubleArrayCost(length); // exactly the same size as a double array diff --git a/src/main/python/tests/matrix/test_print.py b/src/main/python/tests/matrix/test_print.py index 670ce37d072..4dc5a4b4f1c 100644 --- a/src/main/python/tests/matrix/test_print.py +++ b/src/main/python/tests/matrix/test_print.py @@ -46,11 +46,11 @@ def tearDownClass(cls): def test_print_01(self): Matrix(self.sds, np.array([1])).to_string().print().compute() - self.assertEqual('1.000',self.sds.get_stdout()[0]) + self.assertEqual(1,float(self.sds.get_stdout()[0])) def test_print_02(self): self.sds.scalar(1).print().compute() - self.assertEqual('1', self.sds.get_stdout()[0]) + self.assertEqual(1,float(self.sds.get_stdout()[0])) if __name__ == "__main__": unittest.main(exit=False) diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java b/src/test/java/org/apache/sysds/test/TestUtils.java index 624483306bc..48ba4ac3fb1 100644 --- a/src/test/java/org/apache/sysds/test/TestUtils.java +++ b/src/test/java/org/apache/sysds/test/TestUtils.java @@ -1047,10 +1047,12 @@ public static boolean compareMatrices(HashMap m1, HashMap "+v2); - else - System.out.println(e.getKey()+": "+v2+" <--> "+v1); + if(LOG.isDebugEnabled()){ + if(!flag) + LOG.debug(e.getKey()+": "+v1+" <--> "+v2); + else + LOG.debug(e.getKey()+": "+v2+" <--> "+v1); + } } } else { countIdentical++; diff --git a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java index c65ef069694..ac51d7b0754 100644 --- a/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java +++ b/src/test/java/org/apache/sysds/test/component/compress/CompressedTestBase.java @@ -19,6 +19,7 @@ package org.apache.sysds.test.component.compress; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.fail; @@ -114,7 +115,7 @@ public abstract class CompressedTestBase extends TestBase { // .setValidCompressions(EnumSet.of(CompressionType.RLE)).setInvestigateEstimate(true), new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed) - .setValidCompressions(EnumSet.of(CompressionType.SDC)).setInvestigateEstimate(true), + .setValidCompressions(EnumSet.of(CompressionType.SDC)).setInvestigateEstimate(true), // new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed) // .setValidCompressions(EnumSet.of(CompressionType.SDC, CompressionType.DDC)).setInvestigateEstimate(true), @@ -123,9 +124,9 @@ public abstract class CompressedTestBase extends TestBase { // .setInvestigateEstimate(true), // new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setTransposeInput("false") - // .setInvestigateEstimate(true), + // .setInvestigateEstimate(true), // new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setTransposeInput("true") - // .setInvestigateEstimate(true), + // .setInvestigateEstimate(true), // new CompressionSettingsBuilder().setSamplingRatio(0.1).setSeed(compressionSeed).setInvestigateEstimate(true), // new CompressionSettingsBuilder().setSamplingRatio(1.0).setSeed(compressionSeed).setInvestigateEstimate(true) @@ -335,8 +336,8 @@ public void testMatrixMultChain(ChainType ctype) { // LOG.error(ret1); // LOG.error(ret2); // compare result with input - TestUtils.compareMatricesPercentageDistance(DataConverter.convertToDoubleMatrix( - ret1), DataConverter.convertToDoubleMatrix(ret2), 0.9, 0.9, this.toString()); + TestUtils.compareMatricesPercentageDistance(DataConverter + .convertToDoubleMatrix(ret1), DataConverter.convertToDoubleMatrix(ret2), 0.9, 0.9, this.toString()); } catch(Exception e) { @@ -1025,6 +1026,7 @@ public void testSlice(int rl, int ru, int cl, int cu) { return; MatrixBlock ret2 = cmb.slice(rl, ru, cl, cu); MatrixBlock ret1 = mb.slice(rl, ru, cl, cu); + assertEquals(ret1.getNonZeros(), ret2.getNonZeros()); compareResultMatrices(ret1, ret2, 1); } catch(Exception e) { @@ -1059,6 +1061,4 @@ protected void compareResultMatrices(MatrixBlock ret1, MatrixBlock ret2, double compareResultMatrices(d1, d2, toleranceMultiplier); } - - } diff --git a/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java b/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java index 6168265a9a1..9a57989def4 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java +++ b/src/test/java/org/apache/sysds/test/functions/codegenalg/partone/AlgorithmPCA.java @@ -22,7 +22,6 @@ import java.io.File; import java.util.HashMap; -import org.junit.Test; import org.apache.sysds.common.Types.ExecMode; import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.lops.LopProperties.ExecType; @@ -31,6 +30,8 @@ import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; public class AlgorithmPCA extends AutomatedTestBase { @@ -56,81 +57,97 @@ public void setUp() { } @Test + @Ignore public void testPCADenseRewritesCP() { runPCATest(TEST_NAME1, true, false, ExecType.CP, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCASparseRewritesCP() { runPCATest(TEST_NAME1, true, true, ExecType.CP, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCADenseCP() { runPCATest(TEST_NAME1, false, false, ExecType.CP, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCASparseCP() { runPCATest(TEST_NAME1, false, true, ExecType.CP, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCADenseRewritesSP() { runPCATest(TEST_NAME1, true, false, ExecType.SPARK, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCASparseRewritesSP() { runPCATest(TEST_NAME1, true, true, ExecType.SPARK, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCADenseSP() { runPCATest(TEST_NAME1, false, false, ExecType.SPARK, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCASparseSP() { runPCATest(TEST_NAME1, false, true, ExecType.SPARK, CodegenTestType.DEFAULT); } @Test + @Ignore public void testPCADenseRewritesCPFuseAll() { runPCATest(TEST_NAME1, true, false, ExecType.CP, CodegenTestType.FUSE_ALL); } @Test + @Ignore public void testPCASparseRewritesCPFuseAll() { runPCATest(TEST_NAME1, true, true, ExecType.CP, CodegenTestType.FUSE_ALL); } @Test + @Ignore public void testPCADenseRewritesSPFuseAll() { runPCATest(TEST_NAME1, true, false, ExecType.SPARK, CodegenTestType.FUSE_ALL); } @Test + @Ignore public void testPCASparseRewritesSPFuseAll() { runPCATest(TEST_NAME1, true, true, ExecType.SPARK, CodegenTestType.FUSE_ALL); } @Test + @Ignore public void testPCADenseRewritesCPFuseNoRedundancy() { runPCATest(TEST_NAME1, true, false, ExecType.CP, CodegenTestType.FUSE_NO_REDUNDANCY); } @Test + @Ignore public void testPCASparseRewritesCPFuseNoRedundancy() { runPCATest(TEST_NAME1, true, true, ExecType.CP, CodegenTestType.FUSE_NO_REDUNDANCY); } @Test + @Ignore public void testPCADenseRewritesSPFuseNoRedundancy() { runPCATest(TEST_NAME1, true, false, ExecType.SPARK, CodegenTestType.FUSE_NO_REDUNDANCY); } @Test + @Ignore public void testPCASparseRewritesSPFuseNoRedundancy() { runPCATest(TEST_NAME1, true, true, ExecType.SPARK, CodegenTestType.FUSE_NO_REDUNDANCY); } diff --git a/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java b/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java index 940f4f234c5..5e0df61b557 100644 --- a/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java +++ b/src/test/java/org/apache/sysds/test/functions/compress/compressInstructionRewrite.java @@ -67,7 +67,7 @@ public void testCompressInstruction_02() { @Test public void testCompressInstruction_02_toSmallToCompress() { - compressTest(1, 999, 0.2, ExecType.CP, 0, 5, 0, 0, "02"); + compressTest(1, 74, 0.2, ExecType.CP, 0, 5, 0, 0, "02"); } @Test diff --git a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java index 85ac10a03b1..68da2e5a417 100644 --- a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java +++ b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossy.java @@ -24,7 +24,7 @@ public class CompressLossy extends CompressForce { public String TEST_NAME = "compress"; - public String TEST_DIR = "functions/compress/force/lossy/"; + public String TEST_DIR = "functions/compress/force/"; public String TEST_CLASS_DIR = TEST_DIR + CompressLossy.class.getSimpleName() + "/"; private String TEST_CONF = "SystemDS-config-compress-lossy.xml"; private File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); diff --git a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java index 8c7f30903ab..e99e7917040 100644 --- a/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java +++ b/src/test/java/org/apache/sysds/test/functions/compress/configuration/CompressLossyCost.java @@ -24,9 +24,9 @@ public class CompressLossyCost extends CompressCost { public String TEST_NAME = "compress"; - public String TEST_DIR = "functions/compress/cost/lossy"; + public String TEST_DIR = "functions/compress/cost"; public String TEST_CLASS_DIR = TEST_DIR + CompressLossyCost.class.getSimpleName() + "/"; - private String TEST_CONF = "SystemDS-config-compress-lossy-cost.xml"; + private String TEST_CONF = "SystemDS-config-compress-cost-lossy.xml"; private File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); protected String getTestClassDir() { diff --git a/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java index 7fadbe7c0b8..1ee0e4f6509 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/AssertExpressionTest.java @@ -58,6 +58,7 @@ public void testPrintNotExpressionTestRewrite() { */ private void runPrintExpressionTest( String testname, boolean rewrites ) { + setOutputBuffering(true); String TEST_NAME = testname; TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); @@ -82,5 +83,6 @@ private void runPrintExpressionTest( String testname, boolean rewrites ) { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewriteFlag; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java index b911a4abe5d..bec66c3d47b 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ConditionalValidateTest.java @@ -102,6 +102,7 @@ public void testWhileConditionalReadNoErrorNotExists() private void runTest( String testName, Class exceptionClass, boolean fileExists ) { + setOutputBuffering(true); String TEST_NAME = testName; try @@ -135,5 +136,6 @@ private void runTest( String testName, Class exceptionClass, boolean fileExis catch(Exception ex) { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java index 6df7e9cbc88..d524f640700 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeCastingTest.java @@ -87,6 +87,7 @@ public void testMatrixToMatrix() */ private void runTest( String testName, boolean matrixInput, Class exceptionClass ) { + setOutputBuffering(true); String TEST_NAME = testName; int numVals = (exceptionClass != null ? 7 : 1); @@ -134,5 +135,6 @@ else if( testName.equals(TEST_NAME1) ){ { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java index d50eb233075..3b4741a6a70 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/DataTypeChangeTest.java @@ -150,6 +150,7 @@ public void setUp() { private void runTest( String testName, Class exceptionExpected ) { + setOutputBuffering(true); String RI_HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = RI_HOME + testName + ".dml"; programArgs = new String[]{}; @@ -159,6 +160,7 @@ private void runTest( String testName, Class exceptionExpected ) //integration test from outside SystemDS runTest(true, exceptionExpected != null, exceptionExpected, -1); + setOutputBuffering(false); } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java index dee93cd1c4f..9d229fb27a8 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ExistsVariableTest.java @@ -63,12 +63,13 @@ public void testExistsVarNegative() { } private void runExistsTest(String testName, boolean pos) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); String HOME = SCRIPT_DIR + TEST_DIR; String param = pos ? "1" : "0"; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-stats", "-args", param, output("R") }; + programArgs = new String[]{"-args", param, output("R") }; //run script and compare output runTest(true, false, null, -1); @@ -78,5 +79,6 @@ private void runExistsTest(String testName, boolean pos) { val = (val!=null) ? val : 0; Assert.assertTrue("Wrong result: "+param+" vs "+val, val==Double.parseDouble(param)); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java index 7c5cd44dcd7..6d0b260fce8 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionInExpressionTest.java @@ -89,12 +89,13 @@ public void testFunInExpression7() { private void runFunInExpressionTest( String testName ) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testName + ".dml"; - programArgs = new String[]{"-stats", "-args", output("R") }; + programArgs = new String[]{"-args", output("R") }; fullRScriptName = HOME + testName + ".R"; rCmd = getRCmd(expectedDir()); @@ -105,5 +106,6 @@ private void runFunInExpressionTest( String testName ) //compare results double val = readDMLMatrixFromOutputDir("R").get(new CellIndex(1,1)); Assert.assertTrue("Wrong result: 7 vs "+val, Math.abs(val-7) error) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); @@ -58,5 +59,6 @@ private void runFunctionNotFoundTest(String testName, Class error) { programArgs = new String[] {}; runTest(true, true, error, -1); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java b/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java index 60fc339678b..1dd9d622626 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/FunctionReturnTest.java @@ -58,6 +58,8 @@ public void testFunctionReturnBoolIPA() { } private void runFunctionReturnTest( String testname, boolean IPA ) { + + setOutputBuffering(true); boolean oldIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = IPA; try { @@ -66,12 +68,14 @@ private void runFunctionReturnTest( String testname, boolean IPA ) { String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain"}; + programArgs = new String[]{}; runTest(true, false, null, -1); } finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldIPA; } + + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java index 74b60d5ff8d..8d2213cd2b7 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAConstantFoldingScalarVariablePropagationTest.java @@ -94,6 +94,7 @@ public void testConstantFoldingScalarPropagation2NoIPASecondChance() { */ private void runIPAScalarVariablePropagationTest(String testname, boolean IPA_SECOND_CHANCE) { + setOutputBuffering(true); // Save old settings int oldIPANumRep = OptimizerUtils.IPA_NUM_REPETITIONS; boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; @@ -125,5 +126,6 @@ private void runIPAScalarVariablePropagationTest(String testname, boolean IPA_SE DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; rtplatform = platformOld; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java index daf034acde3..35a4d3487e2 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPADeadCodeEliminationTest.java @@ -87,6 +87,7 @@ public void testDeadCodeRemovalFun3IPA() { private void runIPALiteralReplacementTest( String testname, boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try { @@ -106,5 +107,6 @@ private void runIPALiteralReplacementTest( String testname, boolean IPA ) finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java index ca9d1569ba9..83c90ca3a2c 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAFunctionInliningTest.java @@ -122,6 +122,7 @@ public void testFunInline7IPA() { private void runIPAFunInlineTest( String testName, boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -162,5 +163,6 @@ private void runIPAFunInlineTest( String testName, boolean IPA ) finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java index dc832caf868..1a96f52ff84 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPALiteralReplacementTest.java @@ -80,6 +80,7 @@ public void testUnknownRecursionForIfIPA() */ private void runIPALiteralReplacementTest( String testname, boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -107,5 +108,6 @@ private void runIPALiteralReplacementTest( String testname, boolean IPA ) { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java index a57fee227b7..ffbaf2341b6 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPANnzPropagationTest.java @@ -54,6 +54,7 @@ public void testNnzPropgationNegative() { private void runIPANnzPropgationTest(String testname) { + setOutputBuffering(true); // Save old settings boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; ExecMode platformOld = rtplatform; @@ -79,5 +80,6 @@ private void runIPANnzPropgationTest(String testname) DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; rtplatform = platformOld; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java index c89e7db8202..a216cb88609 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarRecursionTest.java @@ -42,6 +42,7 @@ public void setUp() { @Test public void testScalarRecursion() { + setOutputBuffering(true); String TEST_NAME = TEST_NAME1; try @@ -59,5 +60,6 @@ public void testScalarRecursion() { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java index 788e1da59ae..533d9455ad4 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAScalarVariablePropagationTest.java @@ -61,6 +61,8 @@ public void testScalarPropagationIPA() { */ private void runIPAScalarVariablePropagationTest( String testname, boolean IPA ) { + + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -87,5 +89,7 @@ private void runIPAScalarVariablePropagationTest( String testname, boolean IPA ) finally { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + + setOutputBuffering(false); } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java index 24a628dfe86..4d0c1808b10 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/IPAUnknownRecursionTest.java @@ -63,6 +63,7 @@ public void testUnknownRecursionIPA() */ private void runIPAUnknownRecursionTest( boolean IPA ) { + setOutputBuffering(true); boolean oldFlagIPA = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; try @@ -91,5 +92,6 @@ private void runIPAUnknownRecursionTest( boolean IPA ) { OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlagIPA; } + setOutputBuffering(false); } } \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java index 4f7c31aaed5..03598f3ef9f 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ListAndStructTest.java @@ -163,6 +163,7 @@ public void testListIndexingAndCastsRewrites() { private void runListStructTest(String testname, boolean rewrites) { + setOutputBuffering(true); boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; try { @@ -193,5 +194,6 @@ private void runListStructTest(String testname, boolean rewrites) finally { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java index 946ef67b2e1..aa83b4f7fa1 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ListAppendRemove.java @@ -96,6 +96,7 @@ public void testStaticListAppendSpark() { private void runListAppendRemove(String testname, ExecType type, boolean rewrites, boolean conditional) { + setOutputBuffering(true); Types.ExecMode platformOld = setExecMode(type); boolean rewriteOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; @@ -132,5 +133,6 @@ private void runListAppendRemove(String testname, ExecType type, boolean rewrite rtplatform = platformOld; OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewriteOld; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java b/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java index 5423f98827e..e16cc3df98a 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/LongOverflowTest.java @@ -92,6 +92,7 @@ public void testLongOverflowForNoError() */ private void runOverflowTest( String testscript, boolean error ) { + setOutputBuffering(true); String TEST_NAME = testscript; try @@ -113,5 +114,6 @@ private void runOverflowTest( String testscript, boolean error ) { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java b/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java index 7ec6db2b5cf..4ab40638700 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/NegativeLoopIncrementsTest.java @@ -21,12 +21,12 @@ import java.util.HashMap; -import org.junit.Test; import org.apache.sysds.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysds.runtime.meta.MatrixCharacteristics; import org.apache.sysds.test.AutomatedTestBase; import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; +import org.junit.Test; /** * @@ -91,6 +91,7 @@ public void testNegativeParForIncrementsVectorMulti() { private void runNegativeLoopIncrementsTest( String testname, boolean vect, boolean multiStep ) { + setOutputBuffering(true); String TEST_NAME = testname; TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); @@ -118,5 +119,6 @@ private void runNegativeLoopIncrementsTest( String testname, boolean vect, boole //check meta data checkDMLMetaDataFile("R", new MatrixCharacteristics(1,1,1,1)); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java index 4a5c5036892..cd0865e83c7 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/NrowNcolStringTest.java @@ -69,6 +69,7 @@ public void testLengthStringTest() */ private void runNxxStringTest( String testName ) { + setOutputBuffering(true); String TEST_NAME = testName; try @@ -86,5 +87,6 @@ private void runNxxStringTest( String testName ) { throw new RuntimeException(ex); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java b/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java index b05e5dda84f..a4ab57ca9e4 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/PrintExpressionTest.java @@ -70,6 +70,7 @@ public void testPrintMinusExpressionTestRewrite() { */ private void runPrintExpressionTest( String testname, boolean rewrites ) { + setOutputBuffering(true); String TEST_NAME = testname; TestConfiguration config = getTestConfiguration(TEST_NAME); loadTestConfiguration(config); @@ -94,5 +95,6 @@ private void runPrintExpressionTest( String testname, boolean rewrites ) { OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldRewriteFlag; } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java index 9b38e37ee36..a9e303854e6 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/PrintMatrixTest.java @@ -45,6 +45,7 @@ public void testPrintMatrix() { private void runTest( String testName, boolean exceptionExpected ) { + setOutputBuffering(true); TestConfiguration config = getTestConfiguration(TEST_NAME1); loadTestConfiguration(config); @@ -54,5 +55,6 @@ private void runTest( String testName, boolean exceptionExpected ) //run tests runTest(true, exceptionExpected, DMLException.class, -1); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java index fdf2bbca39b..1b2f60965b3 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ReadAfterWriteTest.java @@ -105,6 +105,7 @@ public void testReadAfterWriteScalarAcrossDagNeg() */ private void runReadAfterWriteTest( String testName, boolean positive ) { + setOutputBuffering(true); String TEST_NAME = testName; try @@ -134,5 +135,6 @@ private void runReadAfterWriteTest( String testName, boolean positive ) //cleanup TestUtils.clearDirectory(outputDir()); } + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java index 95b93cff010..b118bd8048e 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ScalarAssignmentTest.java @@ -156,6 +156,7 @@ public void testIfLoopBoolean() */ public void runScalarAssignmentTest( ControlFlowConstruct cfc, ValueType vt ) { + setOutputBuffering(true); String TEST_NAME = null; switch( cfc ) { @@ -188,5 +189,6 @@ public void runScalarAssignmentTest( ControlFlowConstruct cfc, ValueType vt ) boolean exceptionExpected = (cfc==ControlFlowConstruct.PARFOR)? true : false; //dependency analysis int expectedNumberOfJobs = -1; runTest(true, exceptionExpected, LanguageException.class, expectedNumberOfJobs); + setOutputBuffering(false); } } diff --git a/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java b/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java index a5bbd98d9a3..fb22b628ad2 100644 --- a/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java +++ b/src/test/java/org/apache/sysds/test/functions/misc/ZeroRowsColsMatrixTest.java @@ -160,6 +160,7 @@ private void runEmptyMatrixTest( String testname, boolean rewrites, ExecType et private void runEmptyMatrixTest( String testname, boolean rewrites, boolean emptyRet, ExecType et ) { + setOutputBuffering(true); ExecMode platformOld = rtplatform; switch( et ){ case SPARK: rtplatform = ExecMode.SPARK; break; @@ -205,5 +206,6 @@ private void runEmptyMatrixTest( String testname, boolean rewrites, boolean empt rtplatform = platformOld; DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } + setOutputBuffering(false); } } diff --git a/src/test/resources/log4j.properties b/src/test/resources/log4j.properties index 8ea200369e8..e20439eea10 100644 --- a/src/test/resources/log4j.properties +++ b/src/test/resources/log4j.properties @@ -30,8 +30,8 @@ log4j.logger.org.apache.sysds=ERROR # log4j.logger.org.apache.sysds.runtime.instructions.fed=DEBUG # log4j.logger.org.apache.sysds.runtime.compress.CompressedMatrixBlockFactory=DEBUG # log4j.logger.org.apache.sysds.runtime.compress.cocode=DEBUG -log4j.logger.org.apache.sysds.hops.rewrite.RewriteCompressedReblock=DEBUG -log4j.logger.org.apache.sysds.test.functions.compress=DEBUG +# log4j.logger.org.apache.sysds.hops.rewrite.RewriteCompressedReblock=DEBUG +# log4j.logger.org.apache.sysds.test.functions.compress=DEBUG log4j.logger.org.apache.sysds.parser.DataExpression=ERROR log4j.logger.org.apache.spark=OFF log4j.logger.org.apache.hadoop=OFF diff --git a/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml b/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml index 988774ba3cc..bec7a57b92c 100644 --- a/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml +++ b/src/test/scripts/functions/compress/compressInstructionRewrite/SystemDS-config-compress-cost.xml @@ -20,5 +20,5 @@ cost true - cost_scratch_space + target/rewrite_cost_scratch_space diff --git a/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml b/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml index 3e35db17ea0..85793f8a289 100644 --- a/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml +++ b/src/test/scripts/functions/compress/compressScale/SystemDS-config-compress-cost-OLE.xml @@ -20,6 +20,6 @@ cost true - target/cost_scale_scratch_space)OLE + target/cost_scale_scratch_space_OLE OLE diff --git a/src/test/scripts/functions/compress/cost/lossy/SystemDS-config-compress-lossy-cost.xml b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost-lossy.xml similarity index 93% rename from src/test/scripts/functions/compress/cost/lossy/SystemDS-config-compress-lossy-cost.xml rename to src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost-lossy.xml index 097d577d254..1c13c872ec0 100644 --- a/src/test/scripts/functions/compress/cost/lossy/SystemDS-config-compress-lossy-cost.xml +++ b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost-lossy.xml @@ -21,5 +21,5 @@ cost true true - cost_lossy_scratch_space + target/cost_lossy_scratch_space diff --git a/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml index 988774ba3cc..14ff5b8272b 100644 --- a/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml +++ b/src/test/scripts/functions/compress/cost/SystemDS-config-compress-cost.xml @@ -20,5 +20,5 @@ cost true - cost_scratch_space + target/cost_scratch_space diff --git a/src/test/scripts/functions/compress/force/lossy/SystemDS-config-compress-lossy.xml b/src/test/scripts/functions/compress/force/SystemDS-config-compress-lossy.xml similarity index 93% rename from src/test/scripts/functions/compress/force/lossy/SystemDS-config-compress-lossy.xml rename to src/test/scripts/functions/compress/force/SystemDS-config-compress-lossy.xml index 452c56135e8..d517dc5d8bd 100644 --- a/src/test/scripts/functions/compress/force/lossy/SystemDS-config-compress-lossy.xml +++ b/src/test/scripts/functions/compress/force/SystemDS-config-compress-lossy.xml @@ -21,5 +21,5 @@ true true true - lossy_scratch_space + target/force_lossy_scratch_space diff --git a/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml b/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml index b1b68079aee..34f035135b6 100644 --- a/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml +++ b/src/test/scripts/functions/compress/force/SystemDS-config-compress.xml @@ -20,5 +20,5 @@ true true - comp_scratch_space + target/force_comp_scratch_space diff --git a/src/test/scripts/functions/misc/IPANnzPropagation1.dml b/src/test/scripts/functions/misc/IPANnzPropagation1.dml index 711302a1e31..3884c088124 100644 --- a/src/test/scripts/functions/misc/IPANnzPropagation1.dml +++ b/src/test/scripts/functions/misc/IPANnzPropagation1.dml @@ -24,7 +24,7 @@ foo = function(matrix[double] X) return (double sum) { sum = sum(X); } -X = rand(rows=1000, cols=1000000000, sparsity=1e-6) +X = rand(rows=1000, cols=100000000, sparsity=1e-6) s1 = foo(X); s2 = foo(X); print(s1+" "+s2); diff --git a/src/test/scripts/functions/misc/IPANnzPropagation2.dml b/src/test/scripts/functions/misc/IPANnzPropagation2.dml index de0a1df2798..fc4b21381ae 100644 --- a/src/test/scripts/functions/misc/IPANnzPropagation2.dml +++ b/src/test/scripts/functions/misc/IPANnzPropagation2.dml @@ -24,8 +24,8 @@ foo = function(matrix[double] X) return (double sum) { sum = sum(X); } -X = rand(rows=1000, cols=1000000000, sparsity=1e-6) +X = rand(rows=1000, cols=100000000, sparsity=1e-6) s1 = foo(X); -X = rand(rows=1000, cols=1000000000, sparsity=1e-7) +X = rand(rows=1000, cols=100000000, sparsity=1e-7) s2 = foo(X); print(s1+" "+s2);