start to play with some types

src/com/winvector/db/DBDump.java

@@ -4,10 +4,12 @@
 import java.io.FileOutputStream;
 import java.io.PrintStream;
 import java.net.URI;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Date;
 
+import com.winvector.db.DBIterable.RSIterator;
 import com.winvector.db.DBUtil.DBHandle;
 import com.winvector.util.BurstMap;
 import com.winvector.util.TrivialReader;
@@ -17,11 +19,13 @@ public class DBDump {
 
 	public static long runQuery(final String query, final PrintStream p, final DBHandle handle) throws SQLException {
 		final Statement stmt = handle.createReadStatement();
-		final Iterable<BurstMap> source = new DBIterable(stmt,query);
+		final ResultSet rs = stmt.executeQuery(query);
+		final RSIterator source = new RSIterator(rs);
 		boolean first = true;
 		final String sep = "\t";
 		long rowNum = 0;
-		for(final BurstMap row: source) {
+		while(source.hasNext()) {
+			final BurstMap row = source.next();
 			if(first) {
 				boolean firstCol = true;
 				for(final String ki: row.keySet()) {
@@ -30,7 +34,7 @@ public static long runQuery(final String query, final PrintStream p, final DBHan
 					} else {
 						p.print(sep);
 					}
-					p.print(TrivialReader.safeStr(ki));
+					p.print(TrivialReader.safeStr(ki) + ":" + source.getJavaClassName(ki));
 				}
 				p.println();
 				first = false;

src/com/winvector/db/DBIterable.java

@@ -5,6 +5,7 @@
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.NoSuchElementException;
@@ -36,21 +37,27 @@ public static final class RSIterator implements Iterator<BurstMap> {
 		private ResultSet rs;
 		private final String[] colNames;
 		private final int[] colTypes;
+		private final Map<String,String> colNameToJavaClassName = new HashMap<String,String>();
 
 		public RSIterator(final ResultSet rs) throws SQLException {
 			this.rs = rs;
 			if(rs.next()) {
-				ResultSetMetaData meta = rs.getMetaData();
+				final ResultSetMetaData meta = rs.getMetaData();
 				final int n = meta.getColumnCount();
 				final String[] origColNames = new String[n];
 				colTypes = new int[n];
+				final String[] javaClassNames = new String[n];
 				for(int i=0;i<n;++i) {
 					// could also prepend (when appropriate) meta.getTableName(i+1);
 					//origColNames[i] = meta.getColumnName(i+1);
 					origColNames[i] = meta.getColumnLabel(i+1);
 					colTypes[i] = meta.getColumnType(i+1);
+					javaClassNames[i] = meta.getColumnClassName(i+1);
 				}
 				colNames = HBurster.buildHeaderFlds(origColNames);
+				for(int i=0;i<n;++i) {
+					colNameToJavaClassName.put(colNames[i],javaClassNames[i]);
+				}
 			} else {
 				rs.close();
 				this.rs = null;
@@ -60,6 +67,10 @@ public RSIterator(final ResultSet rs) throws SQLException {
 			advance();
 		}
 
+		public String getJavaClassName(final String colName) {
+			return colNameToJavaClassName.get(colName);
+		}
+
 		private void advance() {
 			next = null;
 			if(rs!=null) {
@@ -68,8 +79,11 @@ private void advance() {
 					final Map<String,Object> mp = new TreeMap<String,Object>(ignoreCase);
 					for(int i=0;i<n;++i) {
 						switch(colTypes[i]) {
+						case java.sql.Types.DATE:
+							mp.put(colNames[i],rs.getDate(i+1));
+							break;
 						case java.sql.Types.BIGINT:
-							mp.put(colNames[i],rs.getBigDecimal(i+1));
+							mp.put(colNames[i],rs.getLong(i+1));
 							break;
 						case java.sql.Types.DOUBLE:
 							mp.put(colNames[i],rs.getDouble(i+1));

src/com/winvector/db/LoadTable.java

@@ -134,7 +134,12 @@ private static String stompMarks(final String s) {
 	}
 
 	public static String plumpColumnName(final String kin, final Set<String> seen) {
-		String k = stompMarks(kin).replaceAll("\\W+"," ").trim().replaceAll("\\s+","_");
+		String k = kin;
+		final int colonIndex = k.indexOf(':');
+		if(colonIndex>0) { 		// get rid of any trailing : type info
+			k = k.substring(0,colonIndex);
+		}		
+		k = stompMarks(k).replaceAll("\\W+"," ").trim().replaceAll("\\s+","_");
 		if((k.length()<=0)||invalidColumnNames.contains(k.toLowerCase())||(!Character.isLetter(k.charAt(0)))) {
 			k = columnPrefix + k;
 		}
@@ -248,7 +253,7 @@ public static void loadTable(final Iterable<BurstMap> source, final RowCritique
 		}
 		// set up table
 		final int[] columnTypeCode;
-		final String[] columnTypeName;
+		final String[] columnClassName;
 		{
 			final Statement stmt = handle.conn.createStatement();
 			try {
@@ -261,10 +266,10 @@ public static void loadTable(final Iterable<BurstMap> source, final RowCritique
 			final ResultSet rs = stmt.executeQuery(selectStatement);
 			final ResultSetMetaData rsm = rs.getMetaData();
 			columnTypeCode = new int[sizes.length];
-			columnTypeName = new String[sizes.length];
+			columnClassName = new String[sizes.length];
 			for(int i=0;i<sizes.length;++i) {
 				columnTypeCode[i] = rsm.getColumnType(i+1);
-				columnTypeName[i] = rsm.getColumnTypeName(i+1);
+				columnClassName[i] = rsm.getColumnClassName(i+1);
 			}
 			rs.close();
 			stmt.close();			
@@ -279,18 +284,18 @@ public static void loadTable(final Iterable<BurstMap> source, final RowCritique
 					int i = 0;
 					for(final String k: keys) {
 						if(isInt[i]) {
-							final Double asNumber = row.getAsNumber(k);
-							if(asNumber==null) {
+							final Long asLong = row.getAsLong(k);
+							if(asLong==null) {
 								stmtA.setNull(i+1,columnTypeCode[i]);
 							} else {
-								stmtA.setInt(i+1,(int)Math.round(asNumber));
+								stmtA.setLong(i+1,asLong);
 							}
 						} else if(isNumeric[i]) {	
-							final Double asNumber = row.getAsNumber(k);
-							if(asNumber==null) {
+							final Double asDouble = row.getAsDouble(k);
+							if(asDouble==null) {
 								stmtA.setNull(i+1,columnTypeCode[i]);
 							} else {
-								stmtA.setDouble(i+1,asNumber);
+								stmtA.setDouble(i+1,asDouble);
 							}
 						} else {
 							final String asString = row.getAsString(k);

src/com/winvector/util/BurstMap.java

@@ -31,7 +31,7 @@ public String getAsString(final String key) {
 		return v.toString();
 	}
 
-	public Double getAsNumber(final String key) {
+	public Double getAsDouble(final String key) {
 		final Object v = burst.get(key);
 		if(v==null) {
 			return null;
@@ -46,6 +46,21 @@ public Double getAsNumber(final String key) {
 		}
 	}
 
+	public Long getAsLong(final String key) {
+		final Object v = burst.get(key);
+		if(v==null) {
+			return null;
+		}
+		if(v instanceof Number) {
+			return ((Number)v).longValue();
+		}
+		try {
+			return Long.parseLong(v.toString());
+		} catch (Exception ex) {
+			return null;
+		}
+	}
+
 	@Override
 	public String toString() {
 		final StringBuilder b = new StringBuilder();

src/com/winvector/variables/NumericVariable.java

@@ -34,7 +34,7 @@ public int indexR() {
 
 	@Override
 	public void process(final BurstMap row, final Map<Integer, Double> vec) {
-		final Double v = row.getAsNumber(origColumn);
+		final Double v = row.getAsDouble(origColumn);
 		if(v!=null) {
 			if(v!=0.0) {
 				vec.put(index,v);

src/com/winvector/variables/PrimaVariableInfo.java

@@ -57,7 +57,7 @@ public void trackVariableDefsFromRow(final BurstMap row) {
 			}
 			final String sValue = row.getAsString(key);
 			if(sValue!=null) {
-				final Double nValue = row.getAsNumber(key);
+				final Double nValue = row.getAsDouble(key);
 				if((nValue!=null)&&(!Double.isInfinite(nValue))&&(!Double.isNaN(nValue))) {
 					if (!numericColumnSet.contains(key)) {
 						numericColumnSet.observe(key,nValue,1.0);