/
CommaTest.java
61 lines (50 loc) · 1.97 KB
/
CommaTest.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
package org.test;
import org.junit.Test;
public class CommaTest
{
private static org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger( CommaTest.class );
@Test
public void testOpenCsv() throws Exception
{
log.info( "\nOpenCsv" );
com.opencsv.CSVReader reader = new com.opencsv.CSVReader( new java.io.FileReader("src/test/resources/sample.data") );
String[] record = null;
while( (record = reader.readNext()) != null)
{
log.info( record[0] + "\t" + record[2] + "\t" + record[1] );
}
}
@Test
public void testSpark() throws Exception
{
log.info( "\nspark" );
org.apache.spark.sql.SparkSession spark = org.apache.spark.sql.SparkSession.builder().appName("Simple Application").config("spark.master", "local").getOrCreate();
org.apache.spark.sql.SQLContext sqlContext = new org.apache.spark.sql.SQLContext( spark.sparkContext() );
org.apache.spark.sql.Dataset<org.apache.spark.sql.Row> dataset = sqlContext.read()
.format("com.databricks.spark.csv")
.option("inferSchema", "true")
.option("unescapedQuoteHandling", "STOP_AT_CLOSING_QUOTE" )
.option("header", "true")
.load("src/test/resources/sample.data");
log.info( "Count: " + dataset.count() );
java.util.List<org.apache.spark.sql.Row> rows = dataset.collectAsList();
for( org.apache.spark.sql.Row row : rows )
{
log.info( row.getString( 0 ) + "\t" + row.getString( 2 ) + "\t" + row.getString( 1 ) );
}
}
@Test
public void testCommons() throws Exception
{
log.info( "\ncommons" );
java.io.Reader in = new java.io.FileReader("src/test/resources/sample.data");
Iterable<org.apache.commons.csv.CSVRecord> records = org.apache.commons.csv.CSVFormat.DEFAULT
.withHeader( "h1","h2","h3" )
.withFirstRecordAsHeader()
.parse(in);
for( org.apache.commons.csv.CSVRecord record : records )
{
log.info( record.get( "h1" ) + "\t" + record.get( "h3" ) + "\t" + record.get( "h2" ) );
}
}
}