Skip to content

Commit

Permalink
small fixes on naming
Browse files Browse the repository at this point in the history
  • Loading branch information
robfrank committed Oct 27, 2015
1 parent 4f666e9 commit 4d5e94e
Show file tree
Hide file tree
Showing 7 changed files with 48 additions and 50 deletions.
36 changes: 18 additions & 18 deletions etl/pom.xml
Expand Up @@ -38,18 +38,6 @@


<build> <build>
<plugins> <plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit4</artifactId>
<version>${surefire.version}</version>
</dependency>
</dependencies>
</plugin>

<plugin> <plugin>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId> <artifactId>maven-shade-plugin</artifactId>
Expand Down Expand Up @@ -77,6 +65,18 @@
</execution> </execution>
</executions> </executions>
</plugin> </plugin>
<!--<plugin>-->

<!--<groupId>org.apache.maven.plugins</groupId>-->
<!--<artifactId>maven-surefire-plugin</artifactId>-->
<!--<dependencies>-->
<!--<dependency>-->
<!--<groupId>org.apache.maven.surefire</groupId>-->
<!--<artifactId>surefire-junit4</artifactId>-->
<!--<version>${surefire.version}</version>-->
<!--</dependency>-->
<!--</dependencies>-->
<!--</plugin>-->
</plugins> </plugins>
</build> </build>
<dependencies> <dependencies>
Expand All @@ -85,12 +85,12 @@
<artifactId>orientdb-test-commons</artifactId> <artifactId>orientdb-test-commons</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
<scope>test</scope> <scope>test</scope>
</dependency> <exclusions>
<dependency> <exclusion>
<groupId>junit</groupId> <groupId>org.testng</groupId>
<artifactId>junit</artifactId> <artifactId>testng</artifactId>
<version>4.12</version> </exclusion>
<scope>test</scope> </exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
Expand Down
Expand Up @@ -110,7 +110,7 @@ public static void main(final String[] args) {
processor.execute(); processor.execute();
} }


protected static OETLProcessor parseConfigAndParameters(String[] args) { public static OETLProcessor parseConfigAndParameters(String[] args) {
final OCommandContext context = createDefaultContext(); final OCommandContext context = createDefaultContext();


ODocument configuration = new ODocument().fromJSON("{}"); ODocument configuration = new ODocument().fromJSON("{}");
Expand Down
Expand Up @@ -22,24 +22,22 @@
*/ */
public class OCSVExtractor extends OAbstractSourceExtractor { public class OCSVExtractor extends OAbstractSourceExtractor {


private static String NULL_STRING = "NULL"; private static String NULL_STRING = "NULL";
protected OExtractedItem next; protected OExtractedItem next;
private Map<String, OType> columnTypes = new HashMap<String, OType>(); private Map<String, OType> columnTypes = new HashMap<String, OType>();
private long skipFrom = -1; private long skipFrom = -1;
private long skipTo = -1; private long skipTo = -1;
private Character stringCharacter = '"'; private Character stringCharacter = '"';
private boolean unicode = true; private boolean unicode = true;
private Iterator<CSVRecord> recordIterator; private Iterator<CSVRecord> recordIterator;
private CSVFormat csvFormat; private CSVFormat csvFormat;
private String nullValue = NULL_STRING; private String nullValue = NULL_STRING;
private String dateFormat = "yyyy-MM-dd"; private String dateFormat = "yyyy-MM-dd";


@Override @Override
public ODocument getConfiguration() { public ODocument getConfiguration() {
return new ODocument() return new ODocument().fromJSON(
.fromJSON("{parameters:[" "{parameters:[" + getCommonConfigurationParameters() + ",{separator:{optional:true,description:'Column separator'}},"
+ getCommonConfigurationParameters()
+ ",{separator:{optional:true,description:'Column separator'}},"
+ "{columnsOnFirstLine:{optional:true,description:'Columns are described in the first line'}}," + "{columnsOnFirstLine:{optional:true,description:'Columns are described in the first line'}},"
+ "{columns:{optional:true,description:'Columns array containing names, and optionally type after : (e.g.: name:String, age:int'}}," + "{columns:{optional:true,description:'Columns array containing names, and optionally type after : (e.g.: name:String, age:int'}},"
+ "{nullValue:{optional:true,description:'Value to consider as NULL_STRING. Default is NULL'}}," + "{nullValue:{optional:true,description:'Value to consider as NULL_STRING. Default is NULL'}},"
Expand Down Expand Up @@ -69,15 +67,15 @@ public void configure(OETLProcessor iProcessor, ODocument iConfiguration, OComma
csvFormat = CSVFormat.newFormat(',').withNullString(NULL_STRING).withEscape('\\').withQuote('"'); csvFormat = CSVFormat.newFormat(',').withNullString(NULL_STRING).withEscape('\\').withQuote('"');


if (iConfiguration.containsField("predefinedFormat")) { if (iConfiguration.containsField("predefinedFormat")) {
csvFormat = CSVFormat.valueOf(iConfiguration.<String> field("predefinedFormat").toUpperCase()); csvFormat = CSVFormat.valueOf(iConfiguration.<String>field("predefinedFormat").toUpperCase());
} }


if (iConfiguration.containsField("separator")) { if (iConfiguration.containsField("separator")) {
csvFormat = csvFormat.withDelimiter(iConfiguration.field("separator").toString().charAt(0)); csvFormat = csvFormat.withDelimiter(iConfiguration.field("separator").toString().charAt(0));
} }


if (iConfiguration.containsField("dateFormat")) { if (iConfiguration.containsField("dateFormat")) {
dateFormat = iConfiguration.<String> field("dateFormat"); dateFormat = iConfiguration.<String>field("dateFormat");
} }


if (iConfiguration.containsField("ignoreEmptyLines")) { if (iConfiguration.containsField("ignoreEmptyLines")) {
Expand All @@ -86,8 +84,8 @@ public void configure(OETLProcessor iProcessor, ODocument iConfiguration, OComma
} }


if (iConfiguration.containsField("columnsOnFirstLine")) { if (iConfiguration.containsField("columnsOnFirstLine")) {
boolean columnsOnFirstLine = (Boolean) iConfiguration.field("columnsOnFirstLine"); Boolean columnsOnFirstLine = (Boolean) iConfiguration.field("columnsOnFirstLine");
if (columnsOnFirstLine == true) { if (columnsOnFirstLine.equals(Boolean.TRUE)) {
csvFormat = csvFormat.withHeader(); csvFormat = csvFormat.withHeader();
} }
} else { } else {
Expand Down Expand Up @@ -121,7 +119,7 @@ public void configure(OETLProcessor iProcessor, ODocument iConfiguration, OComma
} }


if (iConfiguration.containsField("nullValue")) { if (iConfiguration.containsField("nullValue")) {
nullValue = iConfiguration.<String> field("nullValue"); nullValue = iConfiguration.<String>field("nullValue");
csvFormat = csvFormat.withNullString(nullValue); csvFormat = csvFormat.withNullString(nullValue);
} }


Expand Down
Expand Up @@ -32,9 +32,9 @@
* @author Luca Garulli * @author Luca Garulli
*/ */
public abstract class ETLBaseTest { public abstract class ETLBaseTest {
protected String[] names = new String[] { "Jay", "Luca", "Bill", "Steve", "Jill", "Luigi", "Enrico", "Emanuele" }; protected String[] names = new String[] { "Jay", "Luca", "Bill", "Steve", "Jill", "Luigi", "Enrico", "Emanuele" };
protected String[] surnames = new String[] { "Miner", "Ferguson", "Cancelli", "Lavori", "Raggio", "Eagles", "Smiles", protected String[] surnames = new String[] { "Miner", "Ferguson", "Cancelli", "Lavori", "Raggio", "Eagles", "Smiles",
"Ironcutter" }; "Ironcutter" };


protected OrientGraph graph; protected OrientGraph graph;
protected OETLProcessor proc; protected OETLProcessor proc;
Expand All @@ -44,7 +44,7 @@ public void setUp() {
graph = new OrientGraph("memory:ETLBaseTest"); graph = new OrientGraph("memory:ETLBaseTest");
graph.setUseLightweightEdges(false); graph.setUseLightweightEdges(false);
proc = new OETLProcessor(); proc = new OETLProcessor();
proc.getFactory().registerLoader(TestLoader.class); proc.getFactory().registerLoader(OETLStubLoader.class);
} }


@After @After
Expand All @@ -53,7 +53,7 @@ public void tearDown() {
} }


protected List<ODocument> getResult() { protected List<ODocument> getResult() {
return ((TestLoader) proc.getLoader()).loadedRecords; return ((OETLStubLoader) proc.getLoader()).loadedRecords;
} }


protected void process(final String cfgJson) { protected void process(final String cfgJson) {
Expand Down
Expand Up @@ -28,14 +28,14 @@
import java.util.List; import java.util.List;


/** /**
* ETL Mock loader to check the result in tests. * ETL Stub loader to check the result in tests.
* *
* @author Luca Garulli on 27/11/14. * @author Luca Garulli on 27/11/14.
*/ */
public class TestLoader extends OAbstractLoader { public class OETLStubLoader extends OAbstractLoader {
public final List<ODocument> loadedRecords = new ArrayList<ODocument>(); public final List<ODocument> loadedRecords = new ArrayList<ODocument>();


public TestLoader() { public OETLStubLoader() {
} }


@Override @Override
Expand Down
Expand Up @@ -32,7 +32,7 @@
* *
* @author Luca Garulli on 27/11/14. * @author Luca Garulli on 27/11/14.
*/ */
public class RandomExtractor extends OAbstractExtractor { public class OETLStubRandomExtractor extends OAbstractExtractor {
private long current = 0; private long current = 0;
private int fields; private int fields;
private long items; private long items;
Expand Down
Expand Up @@ -24,7 +24,7 @@
import com.orientechnologies.orient.core.command.OBasicCommandContext; import com.orientechnologies.orient.core.command.OBasicCommandContext;
import com.orientechnologies.orient.core.record.impl.ODocument; import com.orientechnologies.orient.core.record.impl.ODocument;
import com.orientechnologies.orient.etl.ETLBaseTest; import com.orientechnologies.orient.etl.ETLBaseTest;
import com.orientechnologies.orient.etl.RandomExtractor; import com.orientechnologies.orient.etl.OETLStubRandomExtractor;


import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;


Expand All @@ -39,7 +39,7 @@ public class OJsonRandomExtractorTest extends ETLBaseTest {


@Ignore @Ignore
public void testNonParallel() { public void testNonParallel() {
proc.getFactory().registerExtractor(RandomExtractor.class); proc.getFactory().registerExtractor(OETLStubRandomExtractor.class);


process("{extractor : { random: {items: " + TOTAL + ", fields: 10} }, " process("{extractor : { random: {items: " + TOTAL + ", fields: 10} }, "
+ "loader: { orientdb: { dbURL: 'memory:ETLBaseTest', dbType:'graph', class: 'Person', useLightweightEdges:false, " + "loader: { orientdb: { dbURL: 'memory:ETLBaseTest', dbType:'graph', class: 'Person', useLightweightEdges:false, "
Expand All @@ -56,7 +56,7 @@ public void testNonParallel() {


@Test @Test
public void testParallel() { public void testParallel() {
proc.getFactory().registerExtractor(RandomExtractor.class); proc.getFactory().registerExtractor(OETLStubRandomExtractor.class);


process("{extractor : { random: {items: " + TOTAL + ", fields: 10, delay: 0} }, " process("{extractor : { random: {items: " + TOTAL + ", fields: 10, delay: 0} }, "
+ "loader: { orientdb: { dbURL: 'memory:ETLBaseTest', dbType:'graph', class: 'Person', useLightweightEdges:false, " + "loader: { orientdb: { dbURL: 'memory:ETLBaseTest', dbType:'graph', class: 'Person', useLightweightEdges:false, "
Expand Down

0 comments on commit 4d5e94e

Please sign in to comment.