Skip to content
Permalink
Browse files

add DocumentationArtifacts.createAsJson, createAsCsv (#487)

  • Loading branch information
MykolaGolubyev committed Jan 9, 2020
1 parent 609199d commit 626cf11ff5d5c13854f8c5051bce3a0b31359c23
@@ -32,7 +32,7 @@ class TestToDocExample {
}

private static void validateRules(TableData rules) {
DocumentationArtifacts.create(TestToDocExample,
"account-rules.json", rules.toJson())
DocumentationArtifacts.createAsJson(TestToDocExample, "account-rules.json", rules)
DocumentationArtifacts.createAsCsv(TestToDocExample, "account-rules.csv", rules)
}
}
@@ -64,10 +64,14 @@ public CompositeKey getKey() {
return (E) values.get(idx);
}

public Stream<Object> values() {
public Stream<Object> valuesStream() {
return values.stream();
}

public List<Object> getValues() {
return values;
}

public boolean hasMultiValues() {
return this.hasMultiValues;
}
@@ -97,7 +101,7 @@ public Record evaluateValueGenerators(Record previous, int rowIdx) {
int colIdx = 0;
for (Object value : this.values) {
if (value instanceof TableDataCellValueGenerator) {
newValues.add(((TableDataCellValueGenerator) value).generate(
newValues.add(((TableDataCellValueGenerator<?>) value).generate(
this, previous, rowIdx, colIdx, header.columnNameByIdx(colIdx)));
} else {
newValues.add(value);
@@ -16,13 +16,27 @@

package com.twosigma.webtau.documentation;

import com.twosigma.webtau.data.table.Record;
import com.twosigma.webtau.data.table.TableData;
import com.twosigma.webtau.utils.CsvUtils;
import com.twosigma.webtau.utils.FileUtils;
import com.twosigma.webtau.utils.JsonUtils;

import java.nio.file.Path;

public class DocumentationArtifacts {
public static void create(Class testClass, String artifactName, String textContent) {
public static void create(Class<?> testClass, String artifactName, String text) {
Path path = DocumentationArtifactsLocation.classBasedLocation(testClass).resolve(artifactName);
FileUtils.writeTextContent(path, textContent);
FileUtils.writeTextContent(path, text);
}

public static void createAsJson(Class<?> testClass, String artifactName, TableData tableData) {
create(testClass, artifactName, JsonUtils.serializePrettyPrint(tableData.toListOfMaps()));
}

public static void createAsCsv(Class<?> testClass, String artifactName, TableData tableData) {
create(testClass, artifactName, CsvUtils.serialize(
tableData.getHeader().getNamesStream(),
tableData.rowsStream().map(Record::getValues)));
}
}
@@ -53,8 +53,7 @@ public void shouldGenerateMultipleRowsFromMultiValues() {
TableData tableData = createTableDataWithPermute();

validatePermute(tableData);
DocumentationArtifacts.create(TableDataJavaTest.class, "table-with-permute.json",
tableData.toJson());
DocumentationArtifacts.createAsJson(TableDataJavaTest.class, "table-with-permute.json", tableData);
}

@Test
@@ -134,11 +133,10 @@ private static TableData createTableDataWithAboveRefAndMathExtracted() {
}

private void saveTableWithDate(TableData tableData, String artifactName) {
DocumentationArtifacts.create(TableDataJavaTest.class, artifactName,
DocumentationArtifacts.createAsJson(TableDataJavaTest.class, artifactName,
tableData
.map((rowIdx, colIdx, columnName, v) ->
columnName.equals("Start Date") ?
((LocalDate) v).format(DateTimeFormatter.ISO_DATE) : v)
.toJson());
((LocalDate) v).format(DateTimeFormatter.ISO_DATE) : v));
}
}
@@ -27,11 +27,6 @@
<artifactId>webtau-data</artifactId>

<dependencies>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>

<dependency>
<groupId>com.twosigma.webtau</groupId>
<artifactId>webtau-utils</artifactId>
@@ -17,7 +17,7 @@
package com.twosigma.webtau.data;

import com.twosigma.webtau.cfg.WebTauConfig;
import com.twosigma.webtau.data.csv.CsvParser;
import com.twosigma.webtau.utils.CsvUtils;
import com.twosigma.webtau.utils.FileUtils;
import com.twosigma.webtau.utils.ResourceUtils;

@@ -33,19 +33,19 @@ private Data() {
}

public List<Map<String, String>> csv(String fileOrResourcePath) {
return CsvParser.parse(textContent(fileOrResourcePath));
return CsvUtils.parse(textContent(fileOrResourcePath));
}

public List<Map<String, Object>> csvAutoConverted(String fileOrResourcePath) {
return CsvParser.parseWithAutoConversion(textContent(fileOrResourcePath));
return CsvUtils.parseWithAutoConversion(textContent(fileOrResourcePath));
}

public List<Map<String, String>> csv(List<String> header, String fileOrResourcePath) {
return CsvParser.parse(header, textContent(fileOrResourcePath));
return CsvUtils.parse(header, textContent(fileOrResourcePath));
}

public List<Map<String, Object>> csvAutoConverted(List<String> header, String fileOrResourcePath) {
return CsvParser.parseWithAutoConversion(header, textContent(fileOrResourcePath));
return CsvUtils.parseWithAutoConversion(header, textContent(fileOrResourcePath));
}

private String textContent(String fileOrResourcePath) {
@@ -32,6 +32,11 @@
<artifactId>commons-io</artifactId>
</dependency>

<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-csv</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
@@ -14,21 +14,23 @@
* limitations under the License.
*/

package com.twosigma.webtau.data.csv;
package com.twosigma.webtau.utils;

import com.twosigma.webtau.utils.StringUtils;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVPrinter;
import org.apache.commons.csv.CSVRecord;

import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.*;
import java.util.stream.Stream;

import static java.util.stream.Collectors.toList;

public class CsvParser {
private CsvParser() {
public class CsvUtils {
private CsvUtils() {
}

public static List<Map<String, String>> parse(String content) {
@@ -59,6 +61,22 @@ private CsvParser() {
return convertValues(parse(header, content));
}

public static String serialize(Stream<String> header, Stream<List<Object>> rows) {
try {
StringWriter out = new StringWriter();
CSVPrinter csvPrinter = new CSVPrinter(out, CSVFormat.DEFAULT.withHeader(header.toArray(String[]::new)));

Iterator<List<Object>> it = rows.iterator();
while (it.hasNext()) {
csvPrinter.printRecord(it.next());
}

return out.toString();
} catch (IOException e) {
throw new RuntimeException(e);
}
}

private static CSVParser readCsvRecords(List<String> header, String content) {
try {
CSVFormat csvFormat = CSVFormat.RFC4180;
@@ -95,7 +113,7 @@ private static CSVParser readCsvRecords(List<String> header, String content) {
}

private static List<Map<String, Object>> convertValues(List<Map<String, String>> data) {
return data.stream().map(CsvParser::convertRecord).collect(toList());
return data.stream().map(CsvUtils::convertRecord).collect(toList());
}

private static Map<String, Object> convertRecord(Map<String, String> row) {
@@ -14,48 +14,61 @@
* limitations under the License.
*/

package com.twosigma.webtau.data.csv
package com.twosigma.webtau.utils

import org.junit.Assert
import org.junit.Test

class CsvParserTest {
class CsvUtilsTest {
@Test
void "csv with a header inside content"() {
def csvData = CsvParser.parse("""Account, "Description"
void "parse csv with a header inside content"() {
def csvData = CsvUtils.parse("""Account, "Description"
#12BGD3, "custom, table"
#12BGD3, chair
#91AGB1, lunch
""")

csvData.should == [
assert csvData == [
[Account: "#12BGD3", Description: "custom, table"],
[Account: "#12BGD3", Description: "chair"],
[Account: "#91AGB1", Description: "lunch"]]
}

@Test
void "passed header from outside"() {
def csvData = CsvParser.parse(['Account', 'Description'], """#12BGD3, "custom, table"
void "parse csv with passed header from outside"() {
def csvData = CsvUtils.parse(['Account', 'Description'], """#12BGD3, "custom, table"
#12BGD3, chair
""")

csvData.should == [
assert csvData == [
[Account: "#12BGD3", Description: "custom, table"],
[Account: "#12BGD3", Description: "chair"]]
}

@Test
void "converts text to numbers when using auto conversion method"() {
def csvData = CsvParser.parseWithAutoConversion(['Account', 'Price', 'Description'], """#12BGD3, 100, "custom, table"
def csvData = CsvUtils.parseWithAutoConversion(['Account', 'Price', 'Description'], """#12BGD3, 100, "custom, table"
#12BGD3, 150.5, chair
#12BGD3, 150 %, chair
#12BGD3, "150,000", chair
""")

csvData.should == [
assert csvData == [
[Account: "#12BGD3", Price: 100, Description: "custom, table"],
[Account: "#12BGD3", Price: 150.5, Description: "chair"],
[Account: "#12BGD3", Price: '150 %', Description: "chair"],
[Account: "#12BGD3", Price: 150000, Description: "chair"]]
}

@Test
void "generates csv content from a list of columns and list of rows"() {
def csv = CsvUtils.serialize(["colA", "colB", "colC"].stream(), [
[1, "a", 3],
[null, 4, "hello \"name\""],
].stream())

Assert.assertEquals('colA,colB,colC\r\n' +
'1,a,3\r\n' +
',4,"hello ""name"""\r\n', csv)
}
}

0 comments on commit 626cf11

Please sign in to comment.
You can’t perform that action at this time.