diff --git a/.gitignore b/.gitignore index 02c3ce1..5819785 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .idea/ .gradle/ -build/ -Firehose.iml -test.csv +gradle/ +**build +**.iml diff --git a/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Firehose.java b/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Firehose.java index 8fd7995..a2bd4fd 100644 --- a/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Firehose.java +++ b/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Firehose.java @@ -9,7 +9,9 @@ import com.bryanreinero.firehose.cli.CallBack; import com.bryanreinero.firehose.dao.DataAccessHub; +import com.bryanreinero.firehose.dao.DataStore; import com.bryanreinero.firehose.dao.mongo.MongoDAO; +import com.bryanreinero.firehose.dao.mongo.Read; import com.bryanreinero.firehose.dao.mongo.Write; import com.bryanreinero.firehose.metrics.Interval; import com.bryanreinero.firehose.metrics.SampleSet; @@ -18,6 +20,8 @@ import com.mongodb.MongoClient; import org.bson.Document; +import javax.naming.NamingException; + public class Firehose { private static final String appName = "Firehose"; @@ -29,13 +33,16 @@ public class Firehose { private Converter converter = new Converter(); private BufferedReader br = null; - private final DataAccessHub dataHub; - + private final String SFCabDB = "127.0.0.1:27017"; + private Boolean verbose = false; private String filename = null; private AtomicBoolean running = new AtomicBoolean( true ); + private MongoDAO logDAO + = new MongoDAO( "insert", "SFCabDB", "SFCab.GPSLog", MongoDAO.class ); + private void unitOfWork() { String currentLine = null; @@ -65,10 +72,13 @@ private void unitOfWork() { object = converter.convert(currentLine); } + Write op = new Write( + object, + (MongoDAO) DataAccessHub.INSTANCE.getDescriptor( "insert" ) ); + op.setSamples( app.getSampleSet() ); + // Insert the new Document - app.getThreadPool().submitTask( - new Write(object, (MongoDAO) dataHub.getDescriptor("insert")) - ); + app.getThreadPool().submitTask( op ); } } } @@ -77,7 +87,7 @@ public void parseCommandLineArgs( String[] args ) { app.parseCommandLineArgs( args ); } - public Firehose () { + public Firehose () throws Exception { app = new Application( appName ); @@ -121,11 +131,12 @@ public void handle(String[] values) { stats = new Statistics( samples ); // Initialize the connection to the DB - dataHub = new DataAccessHub(); - dataHub.addCluster( "test", new MongoClient() ); - MongoDAO d = new MongoDAO( "insert", "test", "firehose.csv" ); - d.setSamples( app.getSampleSet() ); - dataHub.addDAO( d ); + DataAccessHub.INSTANCE.setDataStore( + new DataStore( "SFCabDB", appName, SFCabDB, DataStore.Type.mongodb ) + ); + + logDAO.setSamples( app.getSampleSet() ); + DataAccessHub.INSTANCE.setDao( logDAO ); app.addPrinable(this); } diff --git a/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Transformer.java b/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Transformer.java index fd8ff4c..110ca0d 100644 --- a/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Transformer.java +++ b/applications/DSVImport/src/main/java/com/bryanreinero/dsvload/Transformer.java @@ -1,5 +1,6 @@ package com.bryanreinero.dsvload; +import java.util.Date; import java.util.Map; import java.util.HashMap; import java.util.regex.Matcher; @@ -24,6 +25,7 @@ public abstract class Transformer { public static final String TYPE_DOUBLE = "double"; public static final String TYPE_BINARY = "binary"; public static final String TYPE_Object = "object"; + public static final String TYPE_Date = "date"; public static enum Type { ArrayType( Type_Array ), @@ -33,7 +35,8 @@ public static enum Type { FloatType(TYPE_FLOAT), DoubleType(TYPE_DOUBLE), BinaryType(TYPE_BINARY), - ObjectType(TYPE_Object); + ObjectType(TYPE_Object), + DateType(TYPE_Date); private final String name; private Type ( String name ) { this.name = name; } @@ -54,6 +57,8 @@ public static Type getType( String type ) { return BinaryType; if( type.compareTo(TYPE_Object) == 0 ) return BinaryType; + if( type.compareTo(TYPE_Date) == 0 ) + return DateType; return null; } } @@ -140,6 +145,20 @@ public String toString() { } } ); + + transformers.put( TYPE_Date , + new Transformer () { + @Override + public Date transform( String value ) { + return new Date( Long.parseLong( value ) * 1000 ); + } + + @Override + public String toString() { + return TYPE_Date; + } + } + ); } public static Transformer getTransformer( String type ) { diff --git a/build.gradle b/build.gradle index a50f345..489e537 100644 --- a/build.gradle +++ b/build.gradle @@ -3,7 +3,6 @@ group 'bryanreinero' ext.javaDriverVersion = '3.0.0' ext.buildDir = './build' - allprojects { apply plugin: 'java' apply plugin: 'application' @@ -34,8 +33,11 @@ configure(subprojects.findAll { it.name.contains('applications/') }) { project( 'core' ) { + println System.properties['java.home']; + dependencies { testCompile group: 'junit', name: 'junit', version: '4.11' + compile files("${System.properties['java.home']}/../lib/tools.jar") compile "org.mongodb:mongo-java-driver:${javaDriverVersion}" compile "org.apache.commons:commons-math3:3.0" compile "commons-cli:commons-cli:1.2" diff --git a/build.xml b/build.xml index 040f945..2eb8716 100644 --- a/build.xml +++ b/build.xml @@ -1,6 +1,6 @@ - +