From 1a9425765bcfed9ba97b70a2321951f8329f0d3b Mon Sep 17 00:00:00 2001 From: MykolaGolubyev Date: Tue, 2 Apr 2019 16:49:51 -0400 Subject: [PATCH] add cell.above to refer to previous row value (#295) --- .../data/table/TableDataExtensionTest.groovy | 17 +++++ .../main/java/com/twosigma/webtau/Ddjt.java | 3 + .../twosigma/webtau/data/table/Record.java | 67 ++++++++++++++++--- .../twosigma/webtau/data/table/TableData.java | 15 +++-- ...ion.java => TableDataCellMapFunction.java} | 2 +- .../TableDataCellValueGenFullFunction.java | 23 +++++++ .../TableDataCellValueGenFunctions.java | 42 ++++++++++++ ...bleDataCellValueGenOnlyRecordFunction.java | 23 +++++++ .../autogen/TableDataCellValueGenerator.java | 64 ++++++++++++++++++ .../webtau/data/table/TableDataTest.groovy | 21 +++++- .../data/table/TableDataJavaSyntaxTest.java | 42 ++++++++++++ webtau-docs/webtau/reference/table-data.md | 16 ++++- 12 files changed, 318 insertions(+), 17 deletions(-) rename webtau-core/src/main/java/com/twosigma/webtau/data/table/{TableDataCellFunction.java => TableDataCellMapFunction.java} (93%) create mode 100644 webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFullFunction.java create mode 100644 webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFunctions.java create mode 100644 webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenOnlyRecordFunction.java create mode 100644 webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenerator.java create mode 100644 webtau-core/src/test/java/com/twosigma/webtau/data/table/TableDataJavaSyntaxTest.java diff --git a/webtau-core-groovy/src/test/groovy/com/twosigma/webtau/data/table/TableDataExtensionTest.groovy b/webtau-core-groovy/src/test/groovy/com/twosigma/webtau/data/table/TableDataExtensionTest.groovy index 0b0e41e2b..623632d73 100644 --- a/webtau-core-groovy/src/test/groovy/com/twosigma/webtau/data/table/TableDataExtensionTest.groovy +++ b/webtau-core-groovy/src/test/groovy/com/twosigma/webtau/data/table/TableDataExtensionTest.groovy @@ -58,6 +58,15 @@ class TableDataExtensionTest { assert tableData.row(5).toMap() == ['Col A': 'v2a', 'Col B': 20, 'Col C': 'v2c'] } + @Test + void "cell previous should be substituted with value from a previous row"() { + def tableData = createTableDataWithPreviousRef() + assert tableData.numberOfRows() == 3 + assert tableData.row(0).toMap() == ["Col A": "v1a", "Col B": "v1b", "Col C": 10] + assert tableData.row(1).toMap() == ["Col A": "v2a", "Col B": "v2b", "Col C": 10] + assert tableData.row(2).toMap() == ["Col A": "v2a", "Col B": "v2b", "Col C": 20] + } + @Test void "should ignore underscore under header"() { def table = ["hello" | "world"] { @@ -87,6 +96,14 @@ class TableDataExtensionTest { "v2a" | permute(10, 20) | "v2c" } } + static TableData createTableDataWithPreviousRef() { + ["Col A" | "Col B" | "Col C"] { + __________________________________________ + "v1a" | "v1b" | 10 + "v2a" | "v2b" | cell.above + "v2a" | "v2b" | cell.above + 10 } + } + private static void validateTableData(TableData tableData) { tableData.numberOfRows().should == 2 tableData.row(0).toMap().should == ["Col A": "v1a", "Col B": "v1b", "Col C": "v1c"] diff --git a/webtau-core/src/main/java/com/twosigma/webtau/Ddjt.java b/webtau-core/src/main/java/com/twosigma/webtau/Ddjt.java index 0150008c3..59676047f 100644 --- a/webtau-core/src/main/java/com/twosigma/webtau/Ddjt.java +++ b/webtau-core/src/main/java/com/twosigma/webtau/Ddjt.java @@ -18,6 +18,7 @@ import com.twosigma.webtau.data.MultiValue; import com.twosigma.webtau.data.table.TableData; +import com.twosigma.webtau.data.table.autogen.TableDataCellValueGenFunctions; import com.twosigma.webtau.data.table.TableDataUnderscoreOrPlaceholder; import com.twosigma.webtau.expectation.ActualCode; import com.twosigma.webtau.expectation.ActualCodeExpectations; @@ -55,6 +56,8 @@ public class Ddjt { public static final TableDataUnderscoreOrPlaceholder ________________________________________________________________________________ = TableDataUnderscoreOrPlaceholder.INSTANCE; public static final TableDataUnderscoreOrPlaceholder ________________________________________________________________________________________________ = TableDataUnderscoreOrPlaceholder.INSTANCE; + public static final TableDataCellValueGenFunctions cell = new TableDataCellValueGenFunctions(); + public static TableData table(String... columnNames) { return new TableData(Arrays.stream(columnNames)); } diff --git a/webtau-core/src/main/java/com/twosigma/webtau/data/table/Record.java b/webtau-core/src/main/java/com/twosigma/webtau/data/table/Record.java index 0a2013177..4d9830d80 100644 --- a/webtau-core/src/main/java/com/twosigma/webtau/data/table/Record.java +++ b/webtau-core/src/main/java/com/twosigma/webtau/data/table/Record.java @@ -17,24 +17,28 @@ package com.twosigma.webtau.data.table; import com.twosigma.webtau.data.MultiValue; +import com.twosigma.webtau.data.table.autogen.TableDataCellValueGenerator; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.function.Function; import java.util.stream.Stream; -import static java.util.stream.Collectors.toList; - public class Record { private final Header header; private final List values; private final CompositeKey key; + private final boolean hasMultiValues; + private final boolean hasValueGenerators; + public Record(Header header, Stream values) { this.header = header; - this.values = values.collect(toList()); + RecordFromStream recordFromStream = new RecordFromStream(values); + + hasMultiValues = recordFromStream.hasMultiValues; + hasValueGenerators = recordFromStream.hasValueGenerators; + this.values = recordFromStream.values; + this.key = header.hasKeyColumns() ? new CompositeKey(header.getKeyIdxStream().map(this::get)) : null; } @@ -63,7 +67,11 @@ public Stream values() { } public boolean hasMultiValues() { - return this.values.stream().anyMatch(v -> v instanceof MultiValue); + return this.hasMultiValues; + } + + public boolean hasValueGenerators() { + return this.hasValueGenerators; } @SuppressWarnings("unchecked") @@ -78,6 +86,27 @@ public List unwrapMultiValues() { return multiValuesUnwrapper.result; } + public Record evaluateValueGenerators(Record previous, int rowIdx) { + if (!hasValueGenerators()) { + return this; + } + + List newValues = new ArrayList<>(this.values.size()); + int colIdx = 0; + for (Object value : this.values) { + if (value instanceof TableDataCellValueGenerator) { + newValues.add(((TableDataCellValueGenerator) value).generate( + this, previous, rowIdx, colIdx, header.columnNameByIdx(colIdx))); + } else { + newValues.add(value); + } + + colIdx++; + } + + return new Record(header, newValues.stream()); + } + public Map toMap() { Map result = new LinkedHashMap<>(); header.getColumnIdxStream().forEach(i -> result.put(header.columnNameByIdx(i), values.get(i))); @@ -118,4 +147,26 @@ void add(Record record) { result.add(record); } } + + private static class RecordFromStream { + private boolean hasMultiValues; + private boolean hasValueGenerators; + private List values; + + public RecordFromStream(Stream valuesStream) { + values = new ArrayList<>(); + + valuesStream.forEach(v -> { + if (v instanceof MultiValue) { + hasMultiValues = true; + } + + if (v instanceof TableDataCellValueGenerator) { + hasValueGenerators = true; + } + + values.add(v); + }); + } + } } diff --git a/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableData.java b/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableData.java index ec4f5104f..aa4b2e6e0 100644 --- a/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableData.java +++ b/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableData.java @@ -120,18 +120,21 @@ public void addRow(Record record) { int rowIdx = rows.size(); CompositeKey key = getOrBuildKey(rowIdx, record); - Record previous = rowsByKey.put(key, record); - if (previous != null) { + Record existing = rowsByKey.put(key, record); + if (existing != null) { throw new IllegalArgumentException("duplicate entry found with key: " + key + - "\n" + previous + + "\n" + existing + "\n" + record); } + Record previous = rows.isEmpty() ? null : rows.get(rows.size() - 1); + Record withEvaluatedGenerators = record.evaluateValueGenerators(previous, rows.size()); + rowIdxByKey.put(key, rowIdx); - rows.add(record); + rows.add(withEvaluatedGenerators); } - public TableData map(TableDataCellFunction mapper) { + public TableData map(TableDataCellMapFunction mapper) { TableData mapped = new TableData(header); int rowIdx = 0; @@ -150,7 +153,7 @@ public Stream mapColumn(String columnName, Function mapper) { } @SuppressWarnings("unchecked") - private Stream mapRow(int rowIdx, Record originalRow, TableDataCellFunction mapper) { + private Stream mapRow(int rowIdx, Record originalRow, TableDataCellMapFunction mapper) { return header.getColumnIdxStream() .mapToObj(idx -> mapper.apply(rowIdx, idx, header.columnNameByIdx(idx), originalRow.get(idx))); } diff --git a/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableDataCellFunction.java b/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableDataCellMapFunction.java similarity index 93% rename from webtau-core/src/main/java/com/twosigma/webtau/data/table/TableDataCellFunction.java rename to webtau-core/src/main/java/com/twosigma/webtau/data/table/TableDataCellMapFunction.java index 0b6e4b551..11cce383e 100644 --- a/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableDataCellFunction.java +++ b/webtau-core/src/main/java/com/twosigma/webtau/data/table/TableDataCellMapFunction.java @@ -16,6 +16,6 @@ package com.twosigma.webtau.data.table; -public interface TableDataCellFunction { +public interface TableDataCellMapFunction { R apply(int rowIdx, int colIdx, String columnName, T v); } diff --git a/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFullFunction.java b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFullFunction.java new file mode 100644 index 000000000..dfcc02b79 --- /dev/null +++ b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFullFunction.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.twosigma.webtau.data.table.autogen; + +import com.twosigma.webtau.data.table.Record; + +public interface TableDataCellValueGenFullFunction { + R apply(Record row, Record prev, int rowIdx, int colIdx, String columnName); +} diff --git a/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFunctions.java b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFunctions.java new file mode 100644 index 000000000..a554b0b01 --- /dev/null +++ b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenFunctions.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.twosigma.webtau.data.table.autogen; + +import com.twosigma.webtau.data.table.Record; + +/** + * @see com.twosigma.webtau.Ddjt#cell + */ +public class TableDataCellValueGenFunctions { + public final TableDataCellValueGenerator above = new TableDataCellValueGenerator<>(this::previousColumnValue); + + public static TableDataCellValueGenerator value(TableDataCellValueGenFullFunction genFunction) { + return new TableDataCellValueGenerator<>(genFunction); + } + + public static TableDataCellValueGenerator value(TableDataCellValueGenOnlyRecordFunction genFunction) { + return new TableDataCellValueGenerator<>(genFunction); + } + + private R previousColumnValue(Record row, Record prev, int rowIdx, int colIdx, String columnName) { + if (prev == null) { + return null; + } + + return prev.get(columnName); + } +} diff --git a/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenOnlyRecordFunction.java b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenOnlyRecordFunction.java new file mode 100644 index 000000000..3e2857496 --- /dev/null +++ b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenOnlyRecordFunction.java @@ -0,0 +1,23 @@ +/* + * Copyright 2019 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.twosigma.webtau.data.table.autogen; + +import com.twosigma.webtau.data.table.Record; + +public interface TableDataCellValueGenOnlyRecordFunction { + R apply(Record row); +} diff --git a/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenerator.java b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenerator.java new file mode 100644 index 000000000..ed0ea5fce --- /dev/null +++ b/webtau-core/src/main/java/com/twosigma/webtau/data/table/autogen/TableDataCellValueGenerator.java @@ -0,0 +1,64 @@ +/* + * Copyright 2019 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.twosigma.webtau.data.table.autogen; + +import com.twosigma.webtau.data.table.Record; + +public class TableDataCellValueGenerator { + private TableDataCellValueGenFullFunction genFunction; + private TableDataCellValueGenOnlyRecordFunction genOnlyRecordFunction; + + public TableDataCellValueGenerator(TableDataCellValueGenFullFunction genFunction) { + this.genFunction = genFunction; + } + + public TableDataCellValueGenerator(TableDataCellValueGenOnlyRecordFunction genOnlyRecordFunction) { + this.genOnlyRecordFunction = genOnlyRecordFunction; + } + + public Object generate(Record row, Record prev, int rowIdx, int colIdx, String columnName) { + if (genOnlyRecordFunction != null) { + return genOnlyRecordFunction.apply(row); + } else { + return genFunction.apply(row, prev, rowIdx, colIdx, columnName); + } + } + + @SuppressWarnings("unchecked") + public TableDataCellValueGenerator plus(Number v) { + return new TableDataCellValueGenerator(((row, prev, rowIdx, colIdx, columnName) -> { + R calculated = genFunction.apply(row, prev, rowIdx, colIdx, columnName); + return (N) addTwoNumbers((Number) calculated, v); + })); + } + + private static Number addTwoNumbers(Number a, Number b) { + if (b instanceof Double) { + return a.doubleValue() + b.doubleValue(); + } + + if (a instanceof Long || b instanceof Long) { + return a.longValue() + b.longValue(); + } + + if (b instanceof Integer) { + return a.intValue() + b.intValue(); + } + + throw new UnsupportedOperationException(a.getClass() + " + " + b.getClass() + " is not supported"); + } +} diff --git a/webtau-core/src/test/groovy/com/twosigma/webtau/data/table/TableDataTest.groovy b/webtau-core/src/test/groovy/com/twosigma/webtau/data/table/TableDataTest.groovy index c1c3408ee..f3e4c3d7d 100644 --- a/webtau-core/src/test/groovy/com/twosigma/webtau/data/table/TableDataTest.groovy +++ b/webtau-core/src/test/groovy/com/twosigma/webtau/data/table/TableDataTest.groovy @@ -34,6 +34,17 @@ class TableDataTest { validateTableData(tableData) } + @Test + void "cell previous should be substituted with value from a previous row"() { + def tableData = createTableDataWithPreviousRef() + assert tableData.numberOfRows() == 3 + assert tableData.row(0).toMap() == ["Col A": "v1a", "Col B": "v1b", "Col C": 10] + assert tableData.row(1).toMap() == ["Col A": "v2a", "Col B": "v2b", "Col C": 10] + assert tableData.row(2).toMap() == ["Col A": "v2a", "Col B": "v2b", "Col C": 20] + + DocumentationArtifacts.create(TableDataTest, 'table-with-cell-above.json', tableData.toJson()) + } + @Test(expected = IllegalArgumentException) void "should report columns number mismatch during table creation using header and values vararg methods"() { table("Col A", "Col B", "Col C").values( @@ -71,11 +82,19 @@ class TableDataTest { static TableData createTableDataWithPermute() { table("Col A" , "Col B" , "Col C", - ________________________________________________________________, + ________________________________________________________________, permute(true, false), "v1b" , permute('a', 'b'), "v2a" , permute(10, 20) , "v2c") } + static TableData createTableDataWithPreviousRef() { + table("Col A", "Col B", "Col C", + ________________________________________________, + "v1a", "v1b", 10, + "v2a", "v2b", cell.above, + "v2a", "v2b", cell.above.plus(10)) + } + private static void validateTableData(TableData tableData) { assert tableData.numberOfRows() == 2 assert tableData.row(0).toMap() == ["Col A": "v1a", "Col B": "v1b", "Col C": "v1c"] diff --git a/webtau-core/src/test/java/com/twosigma/webtau/data/table/TableDataJavaSyntaxTest.java b/webtau-core/src/test/java/com/twosigma/webtau/data/table/TableDataJavaSyntaxTest.java new file mode 100644 index 000000000..246a3ac66 --- /dev/null +++ b/webtau-core/src/test/java/com/twosigma/webtau/data/table/TableDataJavaSyntaxTest.java @@ -0,0 +1,42 @@ +/* + * Copyright 2019 TWO SIGMA OPEN SOURCE, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.twosigma.webtau.data.table; + +import com.twosigma.webtau.data.table.autogen.TableDataCellValueGenerator; +import org.junit.Test; + +import static com.twosigma.webtau.Ddjt.*; + +public class TableDataJavaSyntaxTest { + @Test + public void makingSureJavaCodeCompiles() { + TableData tableData = createTableDataWithAccessToPrevious(); + actual(tableData.numberOfRows()).should(equal(3)); + } + + // TODO move table data java tests to this test file + // mdoc needs to support removal of "return" and semicolon from function body first + private static TableData createTableDataWithAccessToPrevious() { + TableDataCellValueGenerator increment = cell.above.plus(10); + + return table("Col A", "Col B", "Col C", + ________________________________________________, + "v1a", "v1b", 10, + "v2a", "v2b", increment, + "v2a", "v2b", increment); + } +} diff --git a/webtau-docs/webtau/reference/table-data.md b/webtau-docs/webtau/reference/table-data.md index 7d04e1e2e..bce4cabf9 100644 --- a/webtau-docs/webtau/reference/table-data.md +++ b/webtau-docs/webtau/reference/table-data.md @@ -35,4 +35,18 @@ Java: :include-groovy: com/twosigma/webtau/data/table/TableDataTest.groovy {entry: "createTableDataWithPermute", bodyOnly: true} ``` -:include-table: table-with-permute.json \ No newline at end of file +:include-table: table-with-permute.json + +# Previous Value Reference + +Use `cell.previous` to refer to the previous row value + +```tabs +Groovy: +:include-groovy: com/twosigma/webtau/data/table/TableDataExtensionTest.groovy {entry: "createTableDataWithPreviousRef", bodyOnly: true} + +Java: +:include-groovy: com/twosigma/webtau/data/table/TableDataTest.groovy {entry: "createTableDataWithPreviousRef", bodyOnly: true} +``` + +:include-table: table-with-cell-above.json \ No newline at end of file