[GIO] New Implementation of IOGEN

src/main/java/org/apache/sysds/common/Types.java

@@ -535,6 +535,16 @@ public String toString() {
 		}
 	}
 
+	public enum OpOpGenerateReader {
+		GENERATEREADER;
+		public boolean isGenerateReader(){return  this == GENERATEREADER;}
+
+		@Override
+		public String toString() {
+			return "GRead";
+		}
+	}
+
 	public enum FileFormat {
 		TEXT,   // text cell IJV representation (mm w/o header)
 		MM,     // text matrix market IJV representation
@@ -544,7 +554,8 @@ public enum FileFormat {
 		BINARY, // binary block representation (dense/sparse/ultra-sparse)
 		FEDERATED, // A federated matrix
 		PROTO,  // protocol buffer representation
-		HDF5; // Hierarchical Data Format (HDF)
+		HDF5,  // Hierarchical Data Format (HDF)
+		IOGEN; // Generated Reader
 
 		public boolean isIJV() {
 			return this == TEXT || this == MM;

src/main/java/org/apache/sysds/hops/DataOp.java

@@ -32,6 +32,7 @@
 import org.apache.sysds.conf.ConfigurationManager;
 import org.apache.sysds.hops.rewrite.HopRewriteUtils;
 import org.apache.sysds.lops.Data;
+import org.apache.sysds.lops.DataIOGen;
 import org.apache.sysds.lops.Federated;
 import org.apache.sysds.lops.Lop;
 import org.apache.sysds.common.Types.ExecType;
@@ -54,11 +55,15 @@ public class DataOp extends Hop {
 
 	//read dataop properties
 	private FileFormat _inFormat = FileFormat.TEXT;
+	private String _inIOGenFormat;
 	private long _inBlocksize = -1;
 	private boolean _hasOnlyRDD = false;
 
 	private boolean _recompileRead = true;
 
+	private boolean _ioGenRead = false;
+	private GenerateReaderOp _generateReaderOp;
+
 	/**
 	 * List of "named" input parameters. They are maintained as a hashmap:
 	 * parameter names (String) are mapped as indices (Integer) into getInput()
@@ -247,6 +252,26 @@ public void setFileName(String fn) {
 		_fileName = fn;
 	}
 
+	public void setIOGenRead(boolean isIOGenRead) {
+		_ioGenRead = isIOGenRead;
+	}
+
+	public boolean isIOGenRead(){
+		return _ioGenRead;
+	}
+
+	public String getIOGenFormat() {
+		return _inIOGenFormat;
+	}
+
+	public void setIOGenFormat(String ioGenFormat) {
+		this._inIOGenFormat = ioGenFormat;
+	}
+
+	public void setGenerateReaderOp(GenerateReaderOp op){
+		_generateReaderOp = op;
+	}
+
 	public String getFileName() {
 		return _fileName;
 	}
@@ -283,20 +308,28 @@ public Lop constructLops()
 		for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
 			inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
 		}
+		if(_ioGenRead)
+			inputLops.put("iogenformat", _generateReaderOp.constructLops());
 
 		// Create the lop
 		switch(_op) 
 		{
 			case TRANSIENTREAD:
-				l = new Data(_op, null, inputLops, getName(), null, 
-						getDataType(), getValueType(), getFileFormat());
+				if(!_ioGenRead)
+					l = new Data(_op, null, inputLops, getName(), null, getDataType(), getValueType(), getFileFormat());
+				else
+					l = new DataIOGen(_op, null, inputLops, getName(), null, getDataType(), getValueType(), getIOGenFormat());
 				setOutputDimensions(l);
 				break;
 
 			case PERSISTENTREAD:
-				l = new Data(_op, null, inputLops, getName(), null, 
-						getDataType(), getValueType(), getFileFormat());
-				l.getOutputParameters().setDimensions(getDim1(), getDim2(), _inBlocksize, getNnz(), getUpdateType());
+				if(!_ioGenRead){
+					l = new Data(_op, null, inputLops, getName(), null, getDataType(), getValueType(), getFileFormat());
+					l.getOutputParameters().setDimensions(getDim1(), getDim2(), _inBlocksize, getNnz(), getUpdateType());
+				}
+				else
+					l = new DataIOGen(_op, null, inputLops, getName(), null, getDataType(), getValueType(), getIOGenFormat());
+
 				break;
 
 			case PERSISTENTWRITE:

src/main/java/org/apache/sysds/hops/GenerateReaderOp.java

@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.sysds.hops;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.sysds.common.Types;
+import org.apache.sysds.common.Types.DataType;
+import org.apache.sysds.lops.Lop;
+import org.apache.sysds.lops.ReaderGen;
+import org.apache.sysds.runtime.meta.DataCharacteristics;
+
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+
+public class GenerateReaderOp extends Hop {
+	private static final Log LOG = LogFactory.getLog(GenerateReaderOp.class.getName());
+	private Types.OpOpGenerateReader _op;
+
+	/**
+	 * List of "named" input parameters. They are maintained as a hashmap:
+	 * parameter names (String) are mapped as indices (Integer) into getInput()
+	 * arraylist.
+	 * <p>
+	 * i.e., getInput().get(_paramIndexMap.get(parameterName)) refers to the Hop
+	 * that is associated with parameterName.
+	 */
+	private HashMap<String, Integer> _paramIndexMap = new HashMap<>();
+
+	private GenerateReaderOp() {
+		//default constructor for clone
+	}
+
+	@Override
+	public void checkArity() {
+
+	}
+
+	@Override
+	public boolean allowsAllExecTypes() {
+		return false;
+	}
+
+	@Override
+	protected DataCharacteristics inferOutputCharacteristics(MemoTable memo) {
+		return null;
+	}
+
+	@Override
+	public Lop constructLops() {
+		//return already created lops
+		if( getLops() != null )
+			return getLops();
+
+		Types.ExecType et = Types.ExecType.CP;
+
+
+		// construct lops for all input parameters
+		HashMap<String, Lop> inputLops = new HashMap<>();
+		for (Entry<String, Integer> cur : _paramIndexMap.entrySet()) {
+			inputLops.put(cur.getKey(), getInput().get(cur.getValue()).constructLops());
+		}
+
+		Lop l = new ReaderGen(getInput().get(0).constructLops(),_dataType, _valueType, et, inputLops);
+
+		setLineNumbers(l);
+		setPrivacy(l);
+		setLops(l);
+
+		//add reblock/checkpoint lops if necessary
+		constructAndSetLopsDataFlowProperties();
+
+		return getLops();
+	}
+
+	@Override
+	protected Types.ExecType optFindExecType(boolean transitive) {
+		return null;
+	}
+
+	@Override
+	public String getOpString() {
+		String s = new String("");
+		s += _op.toString();
+		s += " "+getName();
+		return s;
+	}
+
+	@Override
+	public boolean isGPUEnabled() {
+		return false;
+	}
+
+	@Override
+	protected double computeOutputMemEstimate(long dim1, long dim2, long nnz) {
+		return 0;
+	}
+
+	@Override
+	protected double computeIntermediateMemEstimate(long dim1, long dim2, long nnz) {
+		return 0;
+	}
+
+	@Override
+	public void refreshSizeInformation() {
+
+	}
+
+	@Override
+	public Object clone() throws CloneNotSupportedException {
+		return null;
+	}
+
+	@Override
+	public boolean compare(Hop that) {
+		return false;
+	}
+
+	/**
+	 * Generate Reader operation for Matrix
+	 * This constructor supports expression in parameters
+	 * @param l ?
+	 * @param dt              data type
+	 * @param dop             data operator type
+	 * @param in              high-level operator
+	 * @param inputParameters input parameters
+	 */
+	public GenerateReaderOp(String l, DataType dt, Types.OpOpGenerateReader dop, Hop in, HashMap<String, Hop> inputParameters) {
+		_dataType = dt;
+		_op = dop;
+		_name = l;
+		getInput().add(0, in);
+		in.getParent().add(this);
+
+		if(inputParameters != null) {
+			int index = 1;
+			for(Entry<String, Hop> e : inputParameters.entrySet()) {
+				String s = e.getKey();
+				Hop input = e.getValue();
+				getInput().add(input);
+				input.getParent().add(this);
+
+				_paramIndexMap.put(s, index);
+				index++;
+			}
+		}
+	}
+
+	public Types.OpOpGenerateReader getOp() {
+		return _op;
+	}
+}