From e725c68c1e4214159976b3e42c250bb0279e5ab7 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Mon, 2 Jul 2018 22:53:40 +0200 Subject: [PATCH 01/26] #236: Basic, compiling and failing tests for the CurrentIndex ValueExpression. --- .../java/io/parsingdata/metal/Shorthand.java | 2 + .../value/reference/CurrentIndex.java | 55 +++++++++++++++++++ .../value/reference/CurrentIndexTest.java | 50 +++++++++++++++++ 3 files changed, 107 insertions(+) create mode 100644 core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java create mode 100644 core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java diff --git a/core/src/main/java/io/parsingdata/metal/Shorthand.java b/core/src/main/java/io/parsingdata/metal/Shorthand.java index 6fe55d71..9c3f3903 100644 --- a/core/src/main/java/io/parsingdata/metal/Shorthand.java +++ b/core/src/main/java/io/parsingdata/metal/Shorthand.java @@ -44,6 +44,7 @@ import io.parsingdata.metal.expression.value.Cat; import io.parsingdata.metal.expression.value.Const; import io.parsingdata.metal.expression.value.ConstantFactory; +import io.parsingdata.metal.expression.value.reference.CurrentIndex; import io.parsingdata.metal.expression.value.Elvis; import io.parsingdata.metal.expression.value.Expand; import io.parsingdata.metal.expression.value.FoldLeft; @@ -89,6 +90,7 @@ public final class Shorthand { public static final Token EMPTY = def(EMPTY_NAME, 0L); public static final ValueExpression SELF = new Self(); public static final ValueExpression CURRENT_OFFSET = new CurrentOffset(); + public static final ValueExpression CURRENT_INDEX = new CurrentIndex(); public static final Expression TRUE = new True(); private Shorthand() {} diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java new file mode 100644 index 00000000..fdfdd9ef --- /dev/null +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java @@ -0,0 +1,55 @@ +/* + * Copyright 2013-2016 Netherlands Forensic Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.parsingdata.metal.expression.value.reference; + +import java.util.Optional; + +import io.parsingdata.metal.Util; +import io.parsingdata.metal.data.ImmutableList; +import io.parsingdata.metal.data.ParseState; +import io.parsingdata.metal.encoding.Encoding; +import io.parsingdata.metal.expression.value.Value; +import io.parsingdata.metal.expression.value.ValueExpression; + +/** + * A {@link ValueExpression} that represents the current index in an iteration + * (e.g. when inside a {@link io.parsingdata.metal.token.Rep} or + * {@link io.parsingdata.metal.token.RepN}). + */ +public class CurrentIndex implements ValueExpression { + + @Override + public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { + return null; + } + + @Override + public String toString() { + return getClass().getSimpleName(); + } + + @Override + public boolean equals(final Object obj) { + return Util.notNullAndSameClass(this, obj); + } + + @Override + public int hashCode() { + return getClass().hashCode(); + } + +} diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java new file mode 100644 index 00000000..1af83cae --- /dev/null +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java @@ -0,0 +1,50 @@ +/* + * Copyright 2013-2016 Netherlands Forensic Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.parsingdata.metal.expression.value.reference; + +import static io.parsingdata.metal.Shorthand.CURRENT_INDEX; +import static io.parsingdata.metal.Shorthand.con; +import static io.parsingdata.metal.Shorthand.def; +import static io.parsingdata.metal.Shorthand.eq; +import static io.parsingdata.metal.Shorthand.rep; +import static io.parsingdata.metal.Shorthand.repn; +import static io.parsingdata.metal.Shorthand.seq; +import static io.parsingdata.metal.util.EncodingFactory.enc; +import static io.parsingdata.metal.util.ParseStateFactory.stream; + +import java.util.Arrays; +import java.util.Collection; + +import org.junit.runners.Parameterized; + +import io.parsingdata.metal.token.Token; +import io.parsingdata.metal.util.ParameterizedParse; + +public class CurrentIndexTest extends ParameterizedParse { + + public static final Token VALUE_EQ_INDEX = def("value", con(1), eq(CURRENT_INDEX)); + + @Parameterized.Parameters(name="{0} ({4})") + public static Collection data() { + return Arrays.asList(new Object[][] { + { "[0, 1, 2, 3] rep(CURRENT_INDEX)", rep(VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true }, + { "[0, 1, 2, 3] repn(4, CURRENT_INDEX)", repn(VALUE_EQ_INDEX, con(4)), stream(0, 1, 2, 3), enc(), true }, + { "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true } + }); + } + +} From e6b2f3729b7bb0b569da81ada75b7fa1e404897f Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Tue, 3 Jul 2018 09:31:24 +0200 Subject: [PATCH 02/26] #236: Added test for another Token that needs to be included: While. --- .../metal/expression/value/reference/CurrentIndexTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java index 1af83cae..3c781c9b 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java @@ -20,9 +20,11 @@ import static io.parsingdata.metal.Shorthand.con; import static io.parsingdata.metal.Shorthand.def; import static io.parsingdata.metal.Shorthand.eq; +import static io.parsingdata.metal.Shorthand.ltNum; import static io.parsingdata.metal.Shorthand.rep; import static io.parsingdata.metal.Shorthand.repn; import static io.parsingdata.metal.Shorthand.seq; +import static io.parsingdata.metal.Shorthand.whl; import static io.parsingdata.metal.util.EncodingFactory.enc; import static io.parsingdata.metal.util.ParseStateFactory.stream; @@ -43,6 +45,7 @@ public static Collection data() { return Arrays.asList(new Object[][] { { "[0, 1, 2, 3] rep(CURRENT_INDEX)", rep(VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true }, { "[0, 1, 2, 3] repn(4, CURRENT_INDEX)", repn(VALUE_EQ_INDEX, con(4)), stream(0, 1, 2, 3), enc(), true }, + { "[0, 0, 1, 2, 3] def(0), while(<3, CURRENT_INDEX)", seq(def("value", con(1)), whl(VALUE_EQ_INDEX, ltNum(con(3)))), stream(0, 0, 1, 2, 3), enc(), true }, { "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true } }); } From 0e125513a261139cd4bdeb9d1a72c0a85b033158 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 4 Jul 2018 12:20:07 +0200 Subject: [PATCH 03/26] #236: Basic implementation and tests. --- .../value/reference/CurrentIndex.java | 24 ++++++++++++++++++- .../java/io/parsingdata/metal/token/Rep.java | 5 ++++ .../java/io/parsingdata/metal/token/RepN.java | 5 ++++ .../io/parsingdata/metal/token/Token.java | 4 ++++ .../io/parsingdata/metal/token/While.java | 5 ++++ .../value/reference/CurrentIndexTest.java | 6 ++--- 6 files changed, 45 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java index fdfdd9ef..1734b71b 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java @@ -16,10 +16,18 @@ package io.parsingdata.metal.expression.value.reference; +import static java.math.BigInteger.ONE; +import static java.math.BigInteger.ZERO; + +import static io.parsingdata.metal.expression.value.ConstantFactory.createFromNumeric; + +import java.math.BigInteger; import java.util.Optional; import io.parsingdata.metal.Util; import io.parsingdata.metal.data.ImmutableList; +import io.parsingdata.metal.data.ParseGraph; +import io.parsingdata.metal.data.ParseItem; import io.parsingdata.metal.data.ParseState; import io.parsingdata.metal.encoding.Encoding; import io.parsingdata.metal.expression.value.Value; @@ -34,7 +42,21 @@ public class CurrentIndex implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - return null; + final Optional result = findIterable(parseState.order); + if (!result.isPresent()) { return ImmutableList.create(Optional.empty()); } + return ImmutableList.create(Optional.of(createFromNumeric(countIterables(result.get(), ZERO), new Encoding()))); + } + + private Optional findIterable(final ParseItem item) { + if (!item.isGraph() || item.asGraph().isEmpty()) { return Optional.empty(); } + if (!item.getDefinition().isIterable()) { return findIterable(item.asGraph().head); } + return Optional.of(item.asGraph()); + } + + private BigInteger countIterables(final ParseGraph graph, final BigInteger count) { + if (graph.isEmpty()) { return count.subtract(ONE); } + if (graph.tail.getDefinition().equals(graph.getDefinition())) { return countIterables(graph.tail, count.add(ONE)); } + return count.subtract(ONE); } @Override diff --git a/core/src/main/java/io/parsingdata/metal/token/Rep.java b/core/src/main/java/io/parsingdata/metal/token/Rep.java index 222a42a3..2f47b59f 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Rep.java +++ b/core/src/main/java/io/parsingdata/metal/token/Rep.java @@ -59,6 +59,11 @@ private Trampoline> iterate(final Environment environment) .orElseGet(() -> complete(() -> success(environment.parseState.closeBranch()))); } + @Override + public boolean isIterable() { + return true; + } + @Override public String toString() { return getClass().getSimpleName() + "(" + makeNameFragment() + token + ")"; diff --git a/core/src/main/java/io/parsingdata/metal/token/RepN.java b/core/src/main/java/io/parsingdata/metal/token/RepN.java index c8aa58cf..d705231b 100644 --- a/core/src/main/java/io/parsingdata/metal/token/RepN.java +++ b/core/src/main/java/io/parsingdata/metal/token/RepN.java @@ -76,6 +76,11 @@ private Trampoline> iterate(final Environment environment, .orElseGet(() -> complete(Util::failure)); } + @Override + public boolean isIterable() { + return true; + } + @Override public String toString() { return getClass().getSimpleName() + "(" + makeNameFragment() + token + "," + n + ")"; diff --git a/core/src/main/java/io/parsingdata/metal/token/Token.java b/core/src/main/java/io/parsingdata/metal/token/Token.java index 8e5054df..44c5bf62 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Token.java +++ b/core/src/main/java/io/parsingdata/metal/token/Token.java @@ -79,6 +79,10 @@ public boolean isLocal() { return true; } + public boolean isIterable() { + return false; + } + public Token getCanonical(final ParseState parseState) { return this; } diff --git a/core/src/main/java/io/parsingdata/metal/token/While.java b/core/src/main/java/io/parsingdata/metal/token/While.java index 02841cb6..9350f4de 100644 --- a/core/src/main/java/io/parsingdata/metal/token/While.java +++ b/core/src/main/java/io/parsingdata/metal/token/While.java @@ -71,6 +71,11 @@ private Trampoline> iterate(final Environment environment) return complete(() -> success(environment.parseState.closeBranch())); } + @Override + public boolean isIterable() { + return true; + } + @Override public String toString() { return getClass().getSimpleName() + "(" + makeNameFragment() + token + "," + predicate + ")"; diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java index 3c781c9b..2ea1ce20 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java @@ -43,10 +43,10 @@ public class CurrentIndexTest extends ParameterizedParse { @Parameterized.Parameters(name="{0} ({4})") public static Collection data() { return Arrays.asList(new Object[][] { - { "[0, 1, 2, 3] rep(CURRENT_INDEX)", rep(VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true }, + { "[0, 1, 2, 3, 255] rep(CURRENT_INDEX), def(255)", seq(rep(VALUE_EQ_INDEX), def("value", con(1), eq(con(255)))), stream(0, 1, 2, 3, 255), enc(), true }, { "[0, 1, 2, 3] repn(4, CURRENT_INDEX)", repn(VALUE_EQ_INDEX, con(4)), stream(0, 1, 2, 3), enc(), true }, - { "[0, 0, 1, 2, 3] def(0), while(<3, CURRENT_INDEX)", seq(def("value", con(1)), whl(VALUE_EQ_INDEX, ltNum(con(3)))), stream(0, 0, 1, 2, 3), enc(), true }, - { "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true } + { "[255, 0, 1, 2, 3] def(255), while(<3, CURRENT_INDEX)", seq(def("value", con(1), eq(con(255))), whl(VALUE_EQ_INDEX, ltNum(con(3)))), stream(255, 0, 1, 2, 3), enc(), true }, + //{ "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true } }); } From dac4afb17a4281587ad3d5b9df977c49b84a4197 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 4 Jul 2018 22:03:36 +0200 Subject: [PATCH 04/26] #236: Added (failing) test for using CurrentIndex in a nested iterable. --- .../metal/expression/value/reference/CurrentIndexTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java index 2ea1ce20..6132e382 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java @@ -44,8 +44,9 @@ public class CurrentIndexTest extends ParameterizedParse { public static Collection data() { return Arrays.asList(new Object[][] { { "[0, 1, 2, 3, 255] rep(CURRENT_INDEX), def(255)", seq(rep(VALUE_EQ_INDEX), def("value", con(1), eq(con(255)))), stream(0, 1, 2, 3, 255), enc(), true }, - { "[0, 1, 2, 3] repn(4, CURRENT_INDEX)", repn(VALUE_EQ_INDEX, con(4)), stream(0, 1, 2, 3), enc(), true }, - { "[255, 0, 1, 2, 3] def(255), while(<3, CURRENT_INDEX)", seq(def("value", con(1), eq(con(255))), whl(VALUE_EQ_INDEX, ltNum(con(3)))), stream(255, 0, 1, 2, 3), enc(), true }, + { "[0, 1, 2, 3] repn=3(CURRENT_INDEX)", repn(VALUE_EQ_INDEX, con(4)), stream(0, 1, 2, 3), enc(), true }, + { "[255, 0, 1, 2, 3] def(255), while<3(CURRENT_INDEX)", seq(def("value", con(1), eq(con(255))), whl(VALUE_EQ_INDEX, ltNum(con(3)))), stream(255, 0, 1, 2, 3), enc(), true }, + { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_INDEX, repn=3(CURRENT_INDEX))", repn(seq(VALUE_EQ_INDEX, repn(VALUE_EQ_INDEX, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, //{ "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true } }); } From b449cff489b25e5ca8f78176c9720f727897e3b5 Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Tue, 17 Jul 2018 20:30:33 +0200 Subject: [PATCH 05/26] #236: Improved currentIndex implementation by looking for the deepest iterable and counting siblings a long the way. --- .../expression/value/reference/CurrentIndex.java | 15 ++++++++------- .../value/reference/CurrentIndexTest.java | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java index 1734b71b..66d22839 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java @@ -42,15 +42,16 @@ public class CurrentIndex implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - final Optional result = findIterable(parseState.order); - if (!result.isPresent()) { return ImmutableList.create(Optional.empty()); } - return ImmutableList.create(Optional.of(createFromNumeric(countIterables(result.get(), ZERO), new Encoding()))); + final BigInteger currentIndex = findCurrentOffset(parseState.order, ZERO); + return ImmutableList.create(Optional.of(createFromNumeric(currentIndex, new Encoding()))); } - private Optional findIterable(final ParseItem item) { - if (!item.isGraph() || item.asGraph().isEmpty()) { return Optional.empty(); } - if (!item.getDefinition().isIterable()) { return findIterable(item.asGraph().head); } - return Optional.of(item.asGraph()); + private BigInteger findCurrentOffset(final ParseItem item, final BigInteger currentIndex) { + if (!item.isGraph() || item.asGraph().isEmpty()) { return currentIndex; } + if (item.getDefinition().isIterable()) { + return findCurrentOffset(item.asGraph().head, countIterables(item.asGraph(), ZERO)); + } + return findCurrentOffset(item.asGraph().head, currentIndex); } private BigInteger countIterables(final ParseGraph graph, final BigInteger count) { diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java index 6132e382..72487d9f 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java @@ -47,6 +47,7 @@ public static Collection data() { { "[0, 1, 2, 3] repn=3(CURRENT_INDEX)", repn(VALUE_EQ_INDEX, con(4)), stream(0, 1, 2, 3), enc(), true }, { "[255, 0, 1, 2, 3] def(255), while<3(CURRENT_INDEX)", seq(def("value", con(1), eq(con(255))), whl(VALUE_EQ_INDEX, ltNum(con(3)))), stream(255, 0, 1, 2, 3), enc(), true }, { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_INDEX, repn=3(CURRENT_INDEX))", repn(seq(VALUE_EQ_INDEX, repn(VALUE_EQ_INDEX, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, + { "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 0), enc(), true } //{ "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true } }); } From a2949166624e7fd00361d0a3dcdd66e2dadf7c48 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 18 Jul 2018 12:39:47 +0200 Subject: [PATCH 06/26] #236: Renamed CurrentIndex to CurrentIteration to better reflect what it represents. Cleaned up test a bit. --- .../java/io/parsingdata/metal/Shorthand.java | 4 ++-- ...{CurrentIndex.java => CurrentIteration.java} | 15 ++++++++------- ...IndexTest.java => CurrentIterationTest.java} | 17 ++++++++--------- 3 files changed, 18 insertions(+), 18 deletions(-) rename core/src/main/java/io/parsingdata/metal/expression/value/reference/{CurrentIndex.java => CurrentIteration.java} (79%) rename core/src/test/java/io/parsingdata/metal/expression/value/reference/{CurrentIndexTest.java => CurrentIterationTest.java} (57%) diff --git a/core/src/main/java/io/parsingdata/metal/Shorthand.java b/core/src/main/java/io/parsingdata/metal/Shorthand.java index 9c3f3903..2b2fb7b2 100644 --- a/core/src/main/java/io/parsingdata/metal/Shorthand.java +++ b/core/src/main/java/io/parsingdata/metal/Shorthand.java @@ -44,7 +44,7 @@ import io.parsingdata.metal.expression.value.Cat; import io.parsingdata.metal.expression.value.Const; import io.parsingdata.metal.expression.value.ConstantFactory; -import io.parsingdata.metal.expression.value.reference.CurrentIndex; +import io.parsingdata.metal.expression.value.reference.CurrentIteration; import io.parsingdata.metal.expression.value.Elvis; import io.parsingdata.metal.expression.value.Expand; import io.parsingdata.metal.expression.value.FoldLeft; @@ -90,7 +90,7 @@ public final class Shorthand { public static final Token EMPTY = def(EMPTY_NAME, 0L); public static final ValueExpression SELF = new Self(); public static final ValueExpression CURRENT_OFFSET = new CurrentOffset(); - public static final ValueExpression CURRENT_INDEX = new CurrentIndex(); + public static final ValueExpression CURRENT_ITERATION = new CurrentIteration(); public static final Expression TRUE = new True(); private Shorthand() {} diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java similarity index 79% rename from core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java rename to core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index 66d22839..ba0f08de 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIndex.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -34,24 +34,25 @@ import io.parsingdata.metal.expression.value.ValueExpression; /** - * A {@link ValueExpression} that represents the current index in an iteration - * (e.g. when inside a {@link io.parsingdata.metal.token.Rep} or + * A {@link ValueExpression} that represents the current iteration in an + * iterable {@link io.parsingdata.metal.token.Token} (e.g. when inside a + * {@link io.parsingdata.metal.token.Rep} or * {@link io.parsingdata.metal.token.RepN}). */ -public class CurrentIndex implements ValueExpression { +public class CurrentIteration implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - final BigInteger currentIndex = findCurrentOffset(parseState.order, ZERO); + final BigInteger currentIndex = findCurrentIteration(parseState.order, ZERO); return ImmutableList.create(Optional.of(createFromNumeric(currentIndex, new Encoding()))); } - private BigInteger findCurrentOffset(final ParseItem item, final BigInteger currentIndex) { + private BigInteger findCurrentIteration(final ParseItem item, final BigInteger currentIndex) { if (!item.isGraph() || item.asGraph().isEmpty()) { return currentIndex; } if (item.getDefinition().isIterable()) { - return findCurrentOffset(item.asGraph().head, countIterables(item.asGraph(), ZERO)); + return findCurrentIteration(item.asGraph().head, countIterables(item.asGraph(), ZERO)); } - return findCurrentOffset(item.asGraph().head, currentIndex); + return findCurrentIteration(item.asGraph().head, currentIndex); } private BigInteger countIterables(final ParseGraph graph, final BigInteger count) { diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java similarity index 57% rename from core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java rename to core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index 72487d9f..77e18a21 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIndexTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -16,7 +16,7 @@ package io.parsingdata.metal.expression.value.reference; -import static io.parsingdata.metal.Shorthand.CURRENT_INDEX; +import static io.parsingdata.metal.Shorthand.CURRENT_ITERATION; import static io.parsingdata.metal.Shorthand.con; import static io.parsingdata.metal.Shorthand.def; import static io.parsingdata.metal.Shorthand.eq; @@ -36,19 +36,18 @@ import io.parsingdata.metal.token.Token; import io.parsingdata.metal.util.ParameterizedParse; -public class CurrentIndexTest extends ParameterizedParse { +public class CurrentIterationTest extends ParameterizedParse { - public static final Token VALUE_EQ_INDEX = def("value", con(1), eq(CURRENT_INDEX)); + public static final Token VALUE_EQ_ITERATION = def("value", con(1), eq(CURRENT_ITERATION)); @Parameterized.Parameters(name="{0} ({4})") public static Collection data() { return Arrays.asList(new Object[][] { - { "[0, 1, 2, 3, 255] rep(CURRENT_INDEX), def(255)", seq(rep(VALUE_EQ_INDEX), def("value", con(1), eq(con(255)))), stream(0, 1, 2, 3, 255), enc(), true }, - { "[0, 1, 2, 3] repn=3(CURRENT_INDEX)", repn(VALUE_EQ_INDEX, con(4)), stream(0, 1, 2, 3), enc(), true }, - { "[255, 0, 1, 2, 3] def(255), while<3(CURRENT_INDEX)", seq(def("value", con(1), eq(con(255))), whl(VALUE_EQ_INDEX, ltNum(con(3)))), stream(255, 0, 1, 2, 3), enc(), true }, - { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_INDEX, repn=3(CURRENT_INDEX))", repn(seq(VALUE_EQ_INDEX, repn(VALUE_EQ_INDEX, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, - { "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 0), enc(), true } - //{ "[0, 1, 2, 3] seq(CURRENT_INDEX, ...)", seq(VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX, VALUE_EQ_INDEX), stream(0, 1, 2, 3), enc(), true } + { "[0, 1, 2, 3, 255] rep(CURRENT_ITERATION), def(255)", seq(rep(VALUE_EQ_ITERATION), def("value", con(1), eq(con(255)))), stream(0, 1, 2, 3, 255), enc(), true }, + { "[0, 1, 2, 3] repn=3(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, + { "[255, 0, 1, 2, 3] def(255), while<3(CURRENT_ITERATION)", seq(def("value", con(1), eq(con(255))), whl(VALUE_EQ_ITERATION, ltNum(con(3)))), stream(255, 0, 1, 2, 3), enc(), true }, + { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, + { "[0, 0] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 0), enc(), true } }); } From ec291c2749463dca5f209dfb86716e90d4ab93c6 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 18 Jul 2018 14:20:40 +0200 Subject: [PATCH 07/26] #236: Fixed one of the CurrentIteration tests. --- .../expression/value/reference/CurrentIterationTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index 77e18a21..2fc085ec 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -20,7 +20,7 @@ import static io.parsingdata.metal.Shorthand.con; import static io.parsingdata.metal.Shorthand.def; import static io.parsingdata.metal.Shorthand.eq; -import static io.parsingdata.metal.Shorthand.ltNum; +import static io.parsingdata.metal.Shorthand.not; import static io.parsingdata.metal.Shorthand.rep; import static io.parsingdata.metal.Shorthand.repn; import static io.parsingdata.metal.Shorthand.seq; @@ -39,13 +39,14 @@ public class CurrentIterationTest extends ParameterizedParse { public static final Token VALUE_EQ_ITERATION = def("value", con(1), eq(CURRENT_ITERATION)); + public static final Token VALUE_EQ_255 = def("value", con(1), eq(con(255))); @Parameterized.Parameters(name="{0} ({4})") public static Collection data() { return Arrays.asList(new Object[][] { - { "[0, 1, 2, 3, 255] rep(CURRENT_ITERATION), def(255)", seq(rep(VALUE_EQ_ITERATION), def("value", con(1), eq(con(255)))), stream(0, 1, 2, 3, 255), enc(), true }, + { "[0, 1, 2, 3, 255] rep(CURRENT_ITERATION), def(255)", seq(rep(VALUE_EQ_ITERATION), VALUE_EQ_255), stream(0, 1, 2, 3, 255), enc(), true }, { "[0, 1, 2, 3] repn=3(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, - { "[255, 0, 1, 2, 3] def(255), while<3(CURRENT_ITERATION)", seq(def("value", con(1), eq(con(255))), whl(VALUE_EQ_ITERATION, ltNum(con(3)))), stream(255, 0, 1, 2, 3), enc(), true }, + { "[255, 0, 1, 2, 3, 255] def(255), while<3(CURRENT_ITERATION), def (255)", seq(VALUE_EQ_255, whl(VALUE_EQ_ITERATION, not(eq(con(3)))), VALUE_EQ_255), stream(255, 0, 1, 2, 3, 255), enc(), true }, { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, { "[0, 0] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 0), enc(), true } }); From c3c961e35deebf41efc1711099d8b06d3ef70146 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 18 Jul 2018 14:50:36 +0200 Subject: [PATCH 08/26] #236: Simplified implementation of CurrentIteration, which also improves coverage. --- .../metal/expression/value/reference/CurrentIteration.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index ba0f08de..6725048f 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -48,7 +48,7 @@ public ImmutableList> eval(final ParseState parseState, final En } private BigInteger findCurrentIteration(final ParseItem item, final BigInteger currentIndex) { - if (!item.isGraph() || item.asGraph().isEmpty()) { return currentIndex; } + if (!item.isGraph()) { return currentIndex; } if (item.getDefinition().isIterable()) { return findCurrentIteration(item.asGraph().head, countIterables(item.asGraph(), ZERO)); } @@ -56,8 +56,7 @@ private BigInteger findCurrentIteration(final ParseItem item, final BigInteger c } private BigInteger countIterables(final ParseGraph graph, final BigInteger count) { - if (graph.isEmpty()) { return count.subtract(ONE); } - if (graph.tail.getDefinition().equals(graph.getDefinition())) { return countIterables(graph.tail, count.add(ONE)); } + if (!graph.isEmpty()) { return countIterables(graph.tail, count.add(ONE)); } return count.subtract(ONE); } From cab9c3eb2a6f48658e08fe40c35dd5fa72a562d1 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 18 Jul 2018 22:14:57 +0200 Subject: [PATCH 09/26] #236: Improved test coverage for CurrentIteration. --- .../src/test/java/io/parsingdata/metal/AutoEqualityTest.java | 3 ++- core/src/test/java/io/parsingdata/metal/ToStringTest.java | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java b/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java index bec338a8..d221e7e1 100644 --- a/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java +++ b/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java @@ -100,6 +100,7 @@ import io.parsingdata.metal.expression.value.bitwise.ShiftLeft; import io.parsingdata.metal.expression.value.bitwise.ShiftRight; import io.parsingdata.metal.expression.value.reference.Count; +import io.parsingdata.metal.expression.value.reference.CurrentIteration; import io.parsingdata.metal.expression.value.reference.CurrentOffset; import io.parsingdata.metal.expression.value.reference.First; import io.parsingdata.metal.expression.value.reference.Last; @@ -191,7 +192,7 @@ public static Collection data() throws IllegalAccessException, Invocat And.class, Or.class, ShiftLeft.class, ShiftRight.class, Add.class, Div.class, Mod.class, Mul.class, io.parsingdata.metal.expression.value.arithmetic.Sub.class, Cat.class, Nth.class, Elvis.class, FoldLeft.class, FoldRight.class, Const.class, Expand.class, Bytes.class, CurrentOffset.class, - FoldCat.class, + FoldCat.class, CurrentIteration.class, // Expressions Eq.class, EqNum.class, EqStr.class, GtEqNum.class, GtNum.class, LtEqNum.class, LtNum.class, io.parsingdata.metal.expression.logical.And.class, io.parsingdata.metal.expression.logical.Or.class, diff --git a/core/src/test/java/io/parsingdata/metal/ToStringTest.java b/core/src/test/java/io/parsingdata/metal/ToStringTest.java index 3bc8d4b6..9239bc93 100644 --- a/core/src/test/java/io/parsingdata/metal/ToStringTest.java +++ b/core/src/test/java/io/parsingdata/metal/ToStringTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static io.parsingdata.metal.Shorthand.CURRENT_ITERATION; import static io.parsingdata.metal.Shorthand.CURRENT_OFFSET; import static io.parsingdata.metal.Shorthand.SELF; import static io.parsingdata.metal.Shorthand.add; @@ -121,7 +122,7 @@ private Token t() { } private ValueExpression v() { - return fold(foldLeft(foldRight(rev(bytes(neg(add(div(mod(mul(sub(cat(last(ref(n()))), first(nth(exp(ref(n()), con(1)), con(1)))), con(1)), cat(ref(n()), ref(t()))), add(SELF, add(offset(ref(n())), add(CURRENT_OFFSET, count(ref(n())))))), elvis(ref(n()), ref(n())))))), Shorthand::add, ref(n())), Shorthand::add), Shorthand::add, ref(n())); + return fold(foldLeft(foldRight(rev(bytes(neg(add(div(mod(mul(sub(cat(last(ref(n()))), first(nth(exp(ref(n()), con(1)), con(1)))), sub(CURRENT_ITERATION, con(1))), cat(ref(n()), ref(t()))), add(SELF, add(offset(ref(n())), add(CURRENT_OFFSET, count(ref(n())))))), elvis(ref(n()), ref(n())))))), Shorthand::add, ref(n())), Shorthand::add), Shorthand::add, ref(n())); } @Test @@ -148,6 +149,8 @@ public void specialExpressions() { assertEquals("Self", SELF.toString()); assertTrue(v().toString().contains("CurrentOffset")); assertEquals("CurrentOffset", CURRENT_OFFSET.toString()); + assertTrue(v().toString().contains("CurrentIteration")); + assertEquals("CurrentIteration", CURRENT_ITERATION.toString()); } @Test From 34508218b3cc52726b6ebb249f21d5556f3064dd Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 18 Jul 2018 22:38:48 +0200 Subject: [PATCH 10/26] #236: Converted CurrentIteration to use trampolines for recursion. --- .../value/reference/CurrentIteration.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index 6725048f..9f74e10f 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -19,11 +19,14 @@ import static java.math.BigInteger.ONE; import static java.math.BigInteger.ZERO; +import static io.parsingdata.metal.Trampoline.complete; +import static io.parsingdata.metal.Trampoline.intermediate; import static io.parsingdata.metal.expression.value.ConstantFactory.createFromNumeric; import java.math.BigInteger; import java.util.Optional; +import io.parsingdata.metal.Trampoline; import io.parsingdata.metal.Util; import io.parsingdata.metal.data.ImmutableList; import io.parsingdata.metal.data.ParseGraph; @@ -43,21 +46,21 @@ public class CurrentIteration implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - final BigInteger currentIndex = findCurrentIteration(parseState.order, ZERO); + final BigInteger currentIndex = findCurrentIteration(parseState.order, ZERO).computeResult(); return ImmutableList.create(Optional.of(createFromNumeric(currentIndex, new Encoding()))); } - private BigInteger findCurrentIteration(final ParseItem item, final BigInteger currentIndex) { - if (!item.isGraph()) { return currentIndex; } + private Trampoline findCurrentIteration(final ParseItem item, final BigInteger currentIndex) { + if (!item.isGraph()) { return complete(() -> currentIndex); } if (item.getDefinition().isIterable()) { - return findCurrentIteration(item.asGraph().head, countIterables(item.asGraph(), ZERO)); + return intermediate(() -> findCurrentIteration(item.asGraph().head, countIterables(item.asGraph(), ZERO).computeResult())); } - return findCurrentIteration(item.asGraph().head, currentIndex); + return intermediate(() -> findCurrentIteration(item.asGraph().head, currentIndex)); } - private BigInteger countIterables(final ParseGraph graph, final BigInteger count) { - if (!graph.isEmpty()) { return countIterables(graph.tail, count.add(ONE)); } - return count.subtract(ONE); + private Trampoline countIterables(final ParseGraph graph, final BigInteger count) { + if (!graph.isEmpty()) { return intermediate(() -> countIterables(graph.tail, count.add(ONE))); } + return complete(() -> count.subtract(ONE)); } @Override From 0d2401bac5fb373998680b7091cc1247eb0b2af2 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Wed, 18 Jul 2018 22:49:56 +0200 Subject: [PATCH 11/26] #236: Added unit test for CurrentIteration. --- .../expression/value/reference/CurrentIterationTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index 2fc085ec..cfc37f9f 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -45,10 +45,11 @@ public class CurrentIterationTest extends ParameterizedParse { public static Collection data() { return Arrays.asList(new Object[][] { { "[0, 1, 2, 3, 255] rep(CURRENT_ITERATION), def(255)", seq(rep(VALUE_EQ_ITERATION), VALUE_EQ_255), stream(0, 1, 2, 3, 255), enc(), true }, - { "[0, 1, 2, 3] repn=3(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, + { "[0, 1, 2, 3] repn=4(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, { "[255, 0, 1, 2, 3, 255] def(255), while<3(CURRENT_ITERATION), def (255)", seq(VALUE_EQ_255, whl(VALUE_EQ_ITERATION, not(eq(con(3)))), VALUE_EQ_255), stream(255, 0, 1, 2, 3, 255), enc(), true }, { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, - { "[0, 0] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 0), enc(), true } + { "[0, 0] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 0), enc(), true }, + { "[0] CURRENT_ITERATION", VALUE_EQ_ITERATION, stream(0), enc(), true } }); } From ad986347c0f2ddf1c86976433fa8f3f6b5a50440 Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Fri, 20 Jul 2018 21:32:39 +0200 Subject: [PATCH 12/26] #236: Keep track of the last iterable instead of the last iteration and count the iterations of the iterable afterwards. --- .../value/reference/CurrentIteration.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index 9f74e10f..a3f74c09 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -46,20 +46,21 @@ public class CurrentIteration implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - final BigInteger currentIndex = findCurrentIteration(parseState.order, ZERO).computeResult(); - return ImmutableList.create(Optional.of(createFromNumeric(currentIndex, new Encoding()))); + final ParseGraph iterable = findIterable(parseState.order, ParseGraph.EMPTY).computeResult(); + final BigInteger iteration = countIterable(iterable, ZERO).computeResult().max(ZERO); + return ImmutableList.create(Optional.of(createFromNumeric(iteration, new Encoding()))); } - private Trampoline findCurrentIteration(final ParseItem item, final BigInteger currentIndex) { - if (!item.isGraph()) { return complete(() -> currentIndex); } + private Trampoline findIterable(final ParseItem item, final ParseGraph iterableCandidate) { + if (!item.isGraph()) { return complete(() -> iterableCandidate); } if (item.getDefinition().isIterable()) { - return intermediate(() -> findCurrentIteration(item.asGraph().head, countIterables(item.asGraph(), ZERO).computeResult())); + return intermediate(() -> findIterable(item.asGraph().head, item.asGraph())); } - return intermediate(() -> findCurrentIteration(item.asGraph().head, currentIndex)); + return intermediate(() -> findIterable(item.asGraph().head, iterableCandidate)); } - private Trampoline countIterables(final ParseGraph graph, final BigInteger count) { - if (!graph.isEmpty()) { return intermediate(() -> countIterables(graph.tail, count.add(ONE))); } + private Trampoline countIterable(final ParseGraph graph, final BigInteger count) { + if (!graph.isEmpty()) { return intermediate(() -> countIterable(graph.tail, count.add(ONE))); } return complete(() -> count.subtract(ONE)); } From f8f50342938589a30940674b7a9d28210ddf1ed1 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Sat, 21 Jul 2018 17:35:37 +0200 Subject: [PATCH 13/26] #236: Small refactorings. --- .../value/reference/CurrentIteration.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index a3f74c09..29c3926e 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -46,22 +46,22 @@ public class CurrentIteration implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - final ParseGraph iterable = findIterable(parseState.order, ParseGraph.EMPTY).computeResult(); - final BigInteger iteration = countIterable(iterable, ZERO).computeResult().max(ZERO); - return ImmutableList.create(Optional.of(createFromNumeric(iteration, new Encoding()))); + final ParseGraph currentIterable = findCurrentIterable(parseState.order, ParseGraph.EMPTY).computeResult(); + final BigInteger currentIteration = countIterable(currentIterable, ZERO).computeResult(); + return ImmutableList.create(Optional.of(createFromNumeric(currentIteration, new Encoding()))); } - private Trampoline findIterable(final ParseItem item, final ParseGraph iterableCandidate) { + private Trampoline findCurrentIterable(final ParseItem item, final ParseGraph iterableCandidate) { if (!item.isGraph()) { return complete(() -> iterableCandidate); } if (item.getDefinition().isIterable()) { - return intermediate(() -> findIterable(item.asGraph().head, item.asGraph())); + return intermediate(() -> findCurrentIterable(item.asGraph().head, item.asGraph())); } - return intermediate(() -> findIterable(item.asGraph().head, iterableCandidate)); + return intermediate(() -> findCurrentIterable(item.asGraph().head, iterableCandidate)); } private Trampoline countIterable(final ParseGraph graph, final BigInteger count) { if (!graph.isEmpty()) { return intermediate(() -> countIterable(graph.tail, count.add(ONE))); } - return complete(() -> count.subtract(ONE)); + return complete(() -> count.subtract(ONE).max(ZERO)); } @Override From a17967cf0c31e3d522fc001e899dffb31f4ec8ff Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Tue, 24 Jul 2018 22:22:54 +0200 Subject: [PATCH 14/26] #236: Make CurrentIteration return empty value when referencing outside an iterable scope. --- .../metal/expression/value/reference/CurrentIteration.java | 4 +++- .../expression/value/reference/CurrentIterationTest.java | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index 29c3926e..fa3d05e3 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -47,6 +47,8 @@ public class CurrentIteration implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { final ParseGraph currentIterable = findCurrentIterable(parseState.order, ParseGraph.EMPTY).computeResult(); + if (currentIterable.isEmpty()) { return ImmutableList.create(Optional.empty()); } + final BigInteger currentIteration = countIterable(currentIterable, ZERO).computeResult(); return ImmutableList.create(Optional.of(createFromNumeric(currentIteration, new Encoding()))); } @@ -61,7 +63,7 @@ private Trampoline findCurrentIterable(final ParseItem item, final P private Trampoline countIterable(final ParseGraph graph, final BigInteger count) { if (!graph.isEmpty()) { return intermediate(() -> countIterable(graph.tail, count.add(ONE))); } - return complete(() -> count.subtract(ONE).max(ZERO)); + return complete(() -> count.subtract(ONE)); } @Override diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index cfc37f9f..fcad3d86 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -48,8 +48,8 @@ public static Collection data() { { "[0, 1, 2, 3] repn=4(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, { "[255, 0, 1, 2, 3, 255] def(255), while<3(CURRENT_ITERATION), def (255)", seq(VALUE_EQ_255, whl(VALUE_EQ_ITERATION, not(eq(con(3)))), VALUE_EQ_255), stream(255, 0, 1, 2, 3, 255), enc(), true }, { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, - { "[0, 0] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 0), enc(), true }, - { "[0] CURRENT_ITERATION", VALUE_EQ_ITERATION, stream(0), enc(), true } + { "[0, 0] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 0), enc(), false }, + { "[0] CURRENT_ITERATION", VALUE_EQ_ITERATION, stream(0), enc(), false } }); } From b2961aa46acf322c8fb9f44c9fa92f6eb3f08b46 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Tue, 31 Jul 2018 13:17:19 +0200 Subject: [PATCH 15/26] #236: More tests. Unfortunately, no way to more precisely check for empty. --- .../expression/value/reference/CurrentIterationTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index fcad3d86..9f7e6e92 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -39,6 +39,7 @@ public class CurrentIterationTest extends ParameterizedParse { public static final Token VALUE_EQ_ITERATION = def("value", con(1), eq(CURRENT_ITERATION)); + public static final Token VALUE_NOT_EQ_ITERATION = def("value", con(1), not(eq(CURRENT_ITERATION))); public static final Token VALUE_EQ_255 = def("value", con(1), eq(con(255))); @Parameterized.Parameters(name="{0} ({4})") @@ -48,8 +49,10 @@ public static Collection data() { { "[0, 1, 2, 3] repn=4(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, { "[255, 0, 1, 2, 3, 255] def(255), while<3(CURRENT_ITERATION), def (255)", seq(VALUE_EQ_255, whl(VALUE_EQ_ITERATION, not(eq(con(3)))), VALUE_EQ_255), stream(255, 0, 1, 2, 3, 255), enc(), true }, { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, - { "[0, 0] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 0), enc(), false }, - { "[0] CURRENT_ITERATION", VALUE_EQ_ITERATION, stream(0), enc(), false } + { "[0, 1] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 1), enc(), false }, + { "[0] CURRENT_ITERATION", VALUE_EQ_ITERATION, stream(0), enc(), false }, + { "[0, 1] seq(!CURRENT_ITERATION, ...)", seq(VALUE_NOT_EQ_ITERATION, VALUE_NOT_EQ_ITERATION), stream(0, 1), enc(), true }, + { "[0] !CURRENT_ITERATION", VALUE_NOT_EQ_ITERATION, stream(0), enc(), true } }); } From 6fa43c588d7c57acf3bc9c2824f1e1effe649408 Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Mon, 20 Aug 2018 23:47:34 +0200 Subject: [PATCH 16/26] #236: New implementation for current iteration, which is now tracked within ParseState, instead of extracting it out of the ParseGraph. The iteration is now independent on the parse result. --- .gitignore | 1 + .../io/parsingdata/metal/data/ParseState.java | 40 ++++++++---- .../value/reference/CurrentIteration.java | 40 +++--------- .../java/io/parsingdata/metal/token/Cho.java | 2 +- .../metal/token/IterableToken.java | 64 +++++++++++++++++++ .../java/io/parsingdata/metal/token/Post.java | 2 +- .../java/io/parsingdata/metal/token/Pre.java | 2 +- .../java/io/parsingdata/metal/token/Rep.java | 39 ++--------- .../java/io/parsingdata/metal/token/RepN.java | 33 ++-------- .../java/io/parsingdata/metal/token/Seq.java | 2 +- .../java/io/parsingdata/metal/token/Sub.java | 2 +- .../java/io/parsingdata/metal/token/Tie.java | 2 +- .../io/parsingdata/metal/token/While.java | 35 ++-------- .../parsingdata/metal/AutoEqualityTest.java | 4 +- .../io/parsingdata/metal/EqualityTest.java | 4 +- .../value/reference/CurrentIterationTest.java | 10 ++- 16 files changed, 136 insertions(+), 146 deletions(-) create mode 100644 core/src/main/java/io/parsingdata/metal/token/IterableToken.java diff --git a/.gitignore b/.gitignore index e87b1635..9fb3b0a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Maven target/ +out/ # Eclipse .settings/ diff --git a/core/src/main/java/io/parsingdata/metal/data/ParseState.java b/core/src/main/java/io/parsingdata/metal/data/ParseState.java index d8a796b6..e28207b4 100644 --- a/core/src/main/java/io/parsingdata/metal/data/ParseState.java +++ b/core/src/main/java/io/parsingdata/metal/data/ParseState.java @@ -16,6 +16,8 @@ package io.parsingdata.metal.data; +import static java.math.BigInteger.ONE; +import static java.math.BigInteger.TEN; import static java.math.BigInteger.ZERO; import static io.parsingdata.metal.Util.checkNotNegative; @@ -36,15 +38,17 @@ public class ParseState { public final ParseGraph order; public final BigInteger offset; public final Source source; + public final ImmutableList iterations; - public ParseState(final ParseGraph order, final Source source, final BigInteger offset) { + public ParseState(final ParseGraph order, final Source source, final BigInteger offset, final ImmutableList iterations) { this.order = checkNotNull(order, "order"); this.source = checkNotNull(source, "source"); this.offset = checkNotNegative(offset, "offset"); + this.iterations = checkNotNull(iterations, "iteration"); } public static ParseState createFromByteStream(final ByteStream input, final BigInteger offset) { - return new ParseState(ParseGraph.EMPTY, new ByteStreamSource(input), offset); + return new ParseState(ParseGraph.EMPTY, new ByteStreamSource(input), offset, new ImmutableList<>()); } public static ParseState createFromByteStream(final ByteStream input) { @@ -52,27 +56,37 @@ public static ParseState createFromByteStream(final ByteStream input) { } public ParseState addBranch(final Token token) { - return new ParseState(order.addBranch(token), source, offset); + if (token.isIterable()) { + return new ParseState(order.addBranch(token), source, offset, iterations.add(ZERO)); + } + return new ParseState(order.addBranch(token), source, offset, iterations); } - public ParseState closeBranch() { - return new ParseState(order.closeBranch(), source, offset); + public ParseState closeBranch(final Token token) { + if (token.isIterable()) { + return new ParseState(order.closeBranch(), source, offset, iterations.tail); + } + return new ParseState(order.closeBranch(), source, offset, iterations); } public ParseState add(final ParseValue parseValue) { - return new ParseState(order.add(parseValue), source, offset); + return new ParseState(order.add(parseValue), source, offset, iterations); } public ParseState add(final ParseReference parseReference) { - return new ParseState(order.add(parseReference), source, offset); + return new ParseState(order.add(parseReference), source, offset, iterations); + } + + public ParseState iter() { + return new ParseState(order, source, offset, iterations.tail.add(iterations.head.add(ONE))); } public Optional seek(final BigInteger newOffset) { - return newOffset.compareTo(ZERO) >= 0 ? Optional.of(new ParseState(order, source, newOffset)) : Optional.empty(); + return newOffset.compareTo(ZERO) >= 0 ? Optional.of(new ParseState(order, source, newOffset, iterations)) : Optional.empty(); } public ParseState source(final ValueExpression dataExpression, final int index, final ParseState parseState, final Encoding encoding) { - return new ParseState(order, new DataExpressionSource(dataExpression, index, parseState, encoding), ZERO); + return new ParseState(order, new DataExpressionSource(dataExpression, index, parseState, encoding), ZERO, iterations); } public Optional slice(final BigInteger length) { @@ -81,7 +95,8 @@ public Optional slice(final BigInteger length) { @Override public String toString() { - return getClass().getSimpleName() + "(source:" + source + ";offset:" + offset + ";order:" + order + ")"; + final String iterationString = iterations.isEmpty() ? "" : ";iterations:" + iterations.toString(); + return getClass().getSimpleName() + "(source:" + source + ";offset:" + offset + ";order:" + order + iterationString + ")"; } @Override @@ -89,12 +104,13 @@ public boolean equals(final Object obj) { return Util.notNullAndSameClass(this, obj) && Objects.equals(order, ((ParseState)obj).order) && Objects.equals(offset, ((ParseState)obj).offset) - && Objects.equals(source, ((ParseState)obj).source); + && Objects.equals(source, ((ParseState)obj).source) + && Objects.equals(iterations, ((ParseState)obj).iterations); } @Override public int hashCode() { - return Objects.hash(getClass(), order, offset, source); + return Objects.hash(getClass(), order, offset, source, iterations); } } diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index fa3d05e3..ac7ffad3 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2016 Netherlands Forensic Institute + * Copyright 2013-2018 Netherlands Forensic Institute * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,54 +16,32 @@ package io.parsingdata.metal.expression.value.reference; -import static java.math.BigInteger.ONE; -import static java.math.BigInteger.ZERO; - -import static io.parsingdata.metal.Trampoline.complete; -import static io.parsingdata.metal.Trampoline.intermediate; import static io.parsingdata.metal.expression.value.ConstantFactory.createFromNumeric; -import java.math.BigInteger; import java.util.Optional; -import io.parsingdata.metal.Trampoline; import io.parsingdata.metal.Util; import io.parsingdata.metal.data.ImmutableList; -import io.parsingdata.metal.data.ParseGraph; -import io.parsingdata.metal.data.ParseItem; import io.parsingdata.metal.data.ParseState; import io.parsingdata.metal.encoding.Encoding; import io.parsingdata.metal.expression.value.Value; import io.parsingdata.metal.expression.value.ValueExpression; +import io.parsingdata.metal.token.Rep; +import io.parsingdata.metal.token.RepN; +import io.parsingdata.metal.token.Token; /** - * A {@link ValueExpression} that represents the current iteration in an - * iterable {@link io.parsingdata.metal.token.Token} (e.g. when inside a - * {@link io.parsingdata.metal.token.Rep} or - * {@link io.parsingdata.metal.token.RepN}). + * A {@link ValueExpression} that represents the 0-based current iteration in an + * iterable {@link Token} (e.g. when inside a {@link Rep} or {@link RepN}). */ public class CurrentIteration implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - final ParseGraph currentIterable = findCurrentIterable(parseState.order, ParseGraph.EMPTY).computeResult(); - if (currentIterable.isEmpty()) { return ImmutableList.create(Optional.empty()); } - - final BigInteger currentIteration = countIterable(currentIterable, ZERO).computeResult(); - return ImmutableList.create(Optional.of(createFromNumeric(currentIteration, new Encoding()))); - } - - private Trampoline findCurrentIterable(final ParseItem item, final ParseGraph iterableCandidate) { - if (!item.isGraph()) { return complete(() -> iterableCandidate); } - if (item.getDefinition().isIterable()) { - return intermediate(() -> findCurrentIterable(item.asGraph().head, item.asGraph())); + if (parseState.iterations.head == null) { + return ImmutableList.create(Optional.empty()); } - return intermediate(() -> findCurrentIterable(item.asGraph().head, iterableCandidate)); - } - - private Trampoline countIterable(final ParseGraph graph, final BigInteger count) { - if (!graph.isEmpty()) { return intermediate(() -> countIterable(graph.tail, count.add(ONE))); } - return complete(() -> count.subtract(ONE)); + return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head, new Encoding()))); } @Override diff --git a/core/src/main/java/io/parsingdata/metal/token/Cho.java b/core/src/main/java/io/parsingdata/metal/token/Cho.java index a2426307..dc47e71e 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Cho.java +++ b/core/src/main/java/io/parsingdata/metal/token/Cho.java @@ -62,7 +62,7 @@ private Trampoline> iterate(final Environment environment, } return list.head .parse(environment) - .map(result -> complete(() -> success(result.closeBranch()))) + .map(result -> complete(() -> success(result.closeBranch(this)))) .orElseGet(() -> intermediate(() -> iterate(environment, list.tail))); } diff --git a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java new file mode 100644 index 00000000..d2d0d207 --- /dev/null +++ b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java @@ -0,0 +1,64 @@ +package io.parsingdata.metal.token; + +import io.parsingdata.metal.Trampoline; +import io.parsingdata.metal.data.Environment; +import io.parsingdata.metal.data.ParseState; +import io.parsingdata.metal.encoding.Encoding; + +import java.util.Objects; +import java.util.Optional; +import java.util.function.Function; + +import static io.parsingdata.metal.Trampoline.complete; +import static io.parsingdata.metal.Trampoline.intermediate; +import static io.parsingdata.metal.Util.checkNotNull; +import static io.parsingdata.metal.Util.success; + +public abstract class IterableToken extends Token { + + public final Token token; + + IterableToken(String name, final Token token, Encoding encoding) { + super(name, encoding); + this.token = checkNotNull(token, "token"); + } + + protected final Optional parse(final Environment environment, final Function stopCondition, final Function> ifIterationFails) { + return iterate(environment.addBranch(this), stopCondition, ifIterationFails).computeResult(); + } + + /** + * Iteratively parse iterations of the token, given a stop condition and the logic how to handle a failed parse. + * + * @param environment the environment to apply the parse to + * @param stopCondition a function to determine when to stop the iteration + * @param ifIterationFails a function to determine how to handle a failed parse + * @return a trampolined {@code Optional} + */ + private Trampoline> iterate(final Environment environment, final Function stopCondition, final Function> ifIterationFails) { + if (stopCondition.apply(environment)) { + return complete(() -> success(environment.parseState.closeBranch(this))); + } + return token + .parse(environment) + .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState.iter()), stopCondition, ifIterationFails))) + .orElseGet(() -> complete(() -> ifIterationFails.apply(environment))); + } + + @Override + public boolean isIterable() { + return true; + } + + @Override + public boolean equals(final Object obj) { + return super.equals(obj) + && Objects.equals(token, ((IterableToken)obj).token); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), token); + } + +} diff --git a/core/src/main/java/io/parsingdata/metal/token/Post.java b/core/src/main/java/io/parsingdata/metal/token/Post.java index 8c5c4018..78d1f1d6 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Post.java +++ b/core/src/main/java/io/parsingdata/metal/token/Post.java @@ -55,7 +55,7 @@ public Post(final String name, final Token token, final Expression predicate, fi protected Optional parseImpl(final Environment environment) { return token .parse(environment.addBranch(this)) - .map(nextParseState -> predicate.eval(nextParseState, environment.encoding) ? success(nextParseState.closeBranch()) : failure()) + .map(nextParseState -> predicate.eval(nextParseState, environment.encoding) ? success(nextParseState.closeBranch(this)) : failure()) .orElseGet(Util::failure); } diff --git a/core/src/main/java/io/parsingdata/metal/token/Pre.java b/core/src/main/java/io/parsingdata/metal/token/Pre.java index 936d059e..12fc07a7 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Pre.java +++ b/core/src/main/java/io/parsingdata/metal/token/Pre.java @@ -59,7 +59,7 @@ protected Optional parseImpl(final Environment environment) { } return token .parse(environment.addBranch(this)) - .map(resultParseState -> success(resultParseState.closeBranch())) + .map(resultParseState -> success(resultParseState.closeBranch(this))) .orElseGet(Util::failure); } diff --git a/core/src/main/java/io/parsingdata/metal/token/Rep.java b/core/src/main/java/io/parsingdata/metal/token/Rep.java index 2f47b59f..740e04a0 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Rep.java +++ b/core/src/main/java/io/parsingdata/metal/token/Rep.java @@ -16,15 +16,10 @@ package io.parsingdata.metal.token; -import static io.parsingdata.metal.Trampoline.complete; -import static io.parsingdata.metal.Trampoline.intermediate; -import static io.parsingdata.metal.Util.checkNotNull; import static io.parsingdata.metal.Util.success; -import java.util.Objects; import java.util.Optional; -import io.parsingdata.metal.Trampoline; import io.parsingdata.metal.data.Environment; import io.parsingdata.metal.data.ParseState; import io.parsingdata.metal.encoding.Encoding; @@ -38,30 +33,15 @@ * * @see RepN */ -public class Rep extends Token { - - public final Token token; +public class Rep extends IterableToken { public Rep(final String name, final Token token, final Encoding encoding) { - super(name, encoding); - this.token = checkNotNull(token, "token"); - } - - @Override - protected Optional parseImpl(final Environment environment) { - return iterate(environment.addBranch(this)).computeResult(); - } - - private Trampoline> iterate(final Environment environment) { - return token - .parse(environment) - .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState)))) - .orElseGet(() -> complete(() -> success(environment.parseState.closeBranch()))); + super(name, token, encoding); } @Override - public boolean isIterable() { - return true; + protected Optional parseImpl(Environment environment) { + return parse(environment, env -> false, env -> success(env.parseState.closeBranch(this))); } @Override @@ -69,15 +49,4 @@ public String toString() { return getClass().getSimpleName() + "(" + makeNameFragment() + token + ")"; } - @Override - public boolean equals(final Object obj) { - return super.equals(obj) - && Objects.equals(token, ((Rep)obj).token); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), token); - } - } diff --git a/core/src/main/java/io/parsingdata/metal/token/RepN.java b/core/src/main/java/io/parsingdata/metal/token/RepN.java index d705231b..69da009f 100644 --- a/core/src/main/java/io/parsingdata/metal/token/RepN.java +++ b/core/src/main/java/io/parsingdata/metal/token/RepN.java @@ -16,17 +16,13 @@ package io.parsingdata.metal.token; -import static io.parsingdata.metal.Trampoline.complete; -import static io.parsingdata.metal.Trampoline.intermediate; import static io.parsingdata.metal.Util.checkNotNull; import static io.parsingdata.metal.Util.failure; -import static io.parsingdata.metal.Util.success; +import java.math.BigInteger; import java.util.Objects; import java.util.Optional; -import io.parsingdata.metal.Trampoline; -import io.parsingdata.metal.Util; import io.parsingdata.metal.data.Environment; import io.parsingdata.metal.data.ImmutableList; import io.parsingdata.metal.data.ParseState; @@ -46,14 +42,12 @@ * @see Rep * @see ValueExpression */ -public class RepN extends Token { +public class RepN extends IterableToken { - public final Token token; public final ValueExpression n; public RepN(final String name, final Token token, final ValueExpression n, final Encoding encoding) { - super(name, encoding); - this.token = checkNotNull(token, "token"); + super(name, token, encoding); this.n = checkNotNull(n, "n"); } @@ -63,22 +57,8 @@ protected Optional parseImpl(final Environment environment) { if (counts.size != 1 || !counts.head.isPresent()) { return failure(); } - return iterate(environment.addBranch(this), counts.head.get().asNumeric().longValueExact()).computeResult(); - } - - private Trampoline> iterate(final Environment environment, final long count) { - if (count <= 0) { - return complete(() -> success(environment.parseState.closeBranch())); - } - return token - .parse(environment) - .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState), count - 1))) - .orElseGet(() -> complete(Util::failure)); - } - - @Override - public boolean isIterable() { - return true; + final BigInteger count = counts.head.get().asNumeric(); + return parse(environment, env -> env.parseState.iterations.head.compareTo(count) >= 0, env -> failure()); } @Override @@ -89,13 +69,12 @@ public String toString() { @Override public boolean equals(final Object obj) { return super.equals(obj) - && Objects.equals(token, ((RepN)obj).token) && Objects.equals(n, ((RepN)obj).n); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), token, n); + return Objects.hash(super.hashCode(), n); } } diff --git a/core/src/main/java/io/parsingdata/metal/token/Seq.java b/core/src/main/java/io/parsingdata/metal/token/Seq.java index efce4d9d..a943a7d7 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Seq.java +++ b/core/src/main/java/io/parsingdata/metal/token/Seq.java @@ -58,7 +58,7 @@ protected Optional parseImpl(final Environment environment) { private Trampoline> iterate(final Environment environment, final ImmutableList list) { if (list.isEmpty()) { - return complete(() -> success(environment.parseState.closeBranch())); + return complete(() -> success(environment.parseState.closeBranch(this))); } return list.head .parse(environment) diff --git a/core/src/main/java/io/parsingdata/metal/token/Sub.java b/core/src/main/java/io/parsingdata/metal/token/Sub.java index 90912a98..921efc23 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Sub.java +++ b/core/src/main/java/io/parsingdata/metal/token/Sub.java @@ -75,7 +75,7 @@ protected Optional parseImpl(final Environment environment) { private Trampoline> iterate(final Environment environment, final ImmutableList> addresses) { if (addresses.isEmpty()) { - return complete(() -> success(environment.parseState.closeBranch())); + return complete(() -> success(environment.parseState.closeBranch(this))); } return addresses.head .flatMap(address -> parse(environment, address.asNumeric())) diff --git a/core/src/main/java/io/parsingdata/metal/token/Tie.java b/core/src/main/java/io/parsingdata/metal/token/Tie.java index df8e0024..6a579426 100644 --- a/core/src/main/java/io/parsingdata/metal/token/Tie.java +++ b/core/src/main/java/io/parsingdata/metal/token/Tie.java @@ -70,7 +70,7 @@ protected Optional parseImpl(final Environment environment) { private Trampoline> iterate(final Environment environment, final ImmutableList> values, final int index, final ParseState returnParseState) { if (values.isEmpty()) { - return complete(() -> success(new ParseState(environment.parseState.closeBranch().order, returnParseState.source, returnParseState.offset))); + return complete(() -> success(new ParseState(environment.parseState.closeBranch(this).order, returnParseState.source, returnParseState.offset, returnParseState.iterations))); } return values.head .map(value -> token diff --git a/core/src/main/java/io/parsingdata/metal/token/While.java b/core/src/main/java/io/parsingdata/metal/token/While.java index 9350f4de..f3840bb6 100644 --- a/core/src/main/java/io/parsingdata/metal/token/While.java +++ b/core/src/main/java/io/parsingdata/metal/token/While.java @@ -17,16 +17,11 @@ package io.parsingdata.metal.token; import static io.parsingdata.metal.Shorthand.TRUE; -import static io.parsingdata.metal.Trampoline.complete; -import static io.parsingdata.metal.Trampoline.intermediate; -import static io.parsingdata.metal.Util.checkNotNull; -import static io.parsingdata.metal.Util.success; +import static io.parsingdata.metal.Util.failure; import java.util.Objects; import java.util.Optional; -import io.parsingdata.metal.Trampoline; -import io.parsingdata.metal.Util; import io.parsingdata.metal.data.Environment; import io.parsingdata.metal.data.ParseState; import io.parsingdata.metal.encoding.Encoding; @@ -45,35 +40,18 @@ * * @see Expression */ -public class While extends Token { +public class While extends IterableToken { - public final Token token; public final Expression predicate; public While(final String name, final Token token, final Expression predicate, final Encoding encoding) { - super(name, encoding); - this.token = checkNotNull(token, "token"); + super(name, token, encoding); this.predicate = predicate == null ? TRUE : predicate; } @Override - protected Optional parseImpl(final Environment environment) { - return iterate(environment.addBranch(this)).computeResult(); - } - - private Trampoline> iterate(final Environment environment) { - if (predicate.eval(environment.parseState, environment.encoding)) { - return token - .parse(environment) - .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState)))) - .orElseGet(() -> complete(Util::failure)); - } - return complete(() -> success(environment.parseState.closeBranch())); - } - - @Override - public boolean isIterable() { - return true; + protected Optional parseImpl(Environment environment) { + return super.parse(environment, env -> !predicate.eval(env.parseState, env.encoding), env -> failure()); } @Override @@ -84,13 +62,12 @@ public String toString() { @Override public boolean equals(final Object obj) { return super.equals(obj) - && Objects.equals(token, ((While)obj).token) && Objects.equals(predicate, ((While)obj).predicate); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), token, predicate); + return Objects.hash(super.hashCode(), predicate); } } diff --git a/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java b/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java index d221e7e1..fbd70dcd 100644 --- a/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java +++ b/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java @@ -134,7 +134,7 @@ public class AutoEqualityTest { private static final ParseValue PARSE_VALUE = new ParseValue("a", any("a"), createFromBytes(new byte[]{1, 2}), enc()); private static final ParseGraph GRAPH_WITH_REFERENCE = createFromByteStream(DUMMY_STREAM).add(new ParseReference(ZERO, new ConstantSource(new byte[]{1, 2}), any("a"))).order; private static final ParseGraph BRANCHED_GRAPH = createFromByteStream(DUMMY_STREAM).addBranch(any("a")).order; - private static final ParseGraph CLOSED_BRANCHED_GRAPH = createFromByteStream(DUMMY_STREAM).addBranch(any("a")).closeBranch().order; + private static final ParseGraph CLOSED_BRANCHED_GRAPH = createFromByteStream(DUMMY_STREAM).addBranch(any("a")).closeBranch(any("a")).order; private static final List> STRINGS = Arrays.asList(() -> "a", () -> "b"); private static final List> ENCODINGS = Arrays.asList(EncodingFactory::enc, EncodingFactory::signed, EncodingFactory::le, () -> new Encoding(Charset.forName("UTF-8"))); @@ -153,7 +153,7 @@ public class AutoEqualityTest { private static final List> PARSE_ITEMS = Arrays.asList(() -> CLOSED_BRANCHED_GRAPH, () -> ParseGraph.EMPTY, () -> GRAPH_WITH_REFERENCE, () -> createFromByteStream(DUMMY_STREAM).add(PARSE_VALUE).order, () -> createFromByteStream(DUMMY_STREAM).add(PARSE_VALUE).add(PARSE_VALUE).order, () -> BRANCHED_GRAPH); private static final List> BYTE_STREAMS = Arrays.asList(() -> new InMemoryByteStream(new byte[] { 1, 2 }), () -> DUMMY_STREAM); private static final List> BIG_INTEGERS = Arrays.asList(() -> ONE, () -> BigInteger.valueOf(3)); - private static final List> PARSE_STATES = Arrays.asList(() -> createFromByteStream(DUMMY_STREAM), () -> createFromByteStream(DUMMY_STREAM, ONE), () -> new ParseState(GRAPH_WITH_REFERENCE, DUMMY_BYTE_STREAM_SOURCE, TEN)); + private static final List> PARSE_STATES = Arrays.asList(() -> createFromByteStream(DUMMY_STREAM), () -> createFromByteStream(DUMMY_STREAM, ONE), () -> new ParseState(GRAPH_WITH_REFERENCE, DUMMY_BYTE_STREAM_SOURCE, TEN, new ImmutableList<>())); private static final List> IMMUTABLE_LISTS = Arrays.asList(ImmutableList::new, () -> ImmutableList.create("TEST"), () -> ImmutableList.create(1), () -> ImmutableList.create(1).add(2)); private static final Map>> mapping = buildMap(); diff --git a/core/src/test/java/io/parsingdata/metal/EqualityTest.java b/core/src/test/java/io/parsingdata/metal/EqualityTest.java index fa619a16..e7d4b90a 100644 --- a/core/src/test/java/io/parsingdata/metal/EqualityTest.java +++ b/core/src/test/java/io/parsingdata/metal/EqualityTest.java @@ -173,8 +173,8 @@ public void parseGraph() { assertFalse(object.equals(null)); assertFalse(object.equals("a")); final ParseState parseState = createFromByteStream(DUMMY_STREAM); - assertNotEquals(parseState.addBranch(any("a")).add(value).add(value).closeBranch().addBranch(any("a")).order, parseState.addBranch(any("a")).closeBranch().addBranch(any("a")).order); - assertNotEquals(parseState.addBranch(any("a")).order, parseState.addBranch(any("a")).closeBranch().order); + assertNotEquals(parseState.addBranch(any("a")).add(value).add(value).closeBranch(any("a")).addBranch(any("a")).order, parseState.addBranch(any("a")).closeBranch(any("a")).addBranch(any("a")).order); + assertNotEquals(parseState.addBranch(any("a")).order, parseState.addBranch(any("a")).closeBranch(any("a")).order); assertNotEquals(parseState.addBranch(any("a")).order, parseState.addBranch(any("b")).order); } diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index 9f7e6e92..1541c4fa 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -17,10 +17,14 @@ package io.parsingdata.metal.expression.value.reference; import static io.parsingdata.metal.Shorthand.CURRENT_ITERATION; +import static io.parsingdata.metal.Shorthand.add; import static io.parsingdata.metal.Shorthand.con; import static io.parsingdata.metal.Shorthand.def; import static io.parsingdata.metal.Shorthand.eq; +import static io.parsingdata.metal.Shorthand.eqNum; import static io.parsingdata.metal.Shorthand.not; +import static io.parsingdata.metal.Shorthand.nth; +import static io.parsingdata.metal.Shorthand.ref; import static io.parsingdata.metal.Shorthand.rep; import static io.parsingdata.metal.Shorthand.repn; import static io.parsingdata.metal.Shorthand.seq; @@ -48,11 +52,13 @@ public static Collection data() { { "[0, 1, 2, 3, 255] rep(CURRENT_ITERATION), def(255)", seq(rep(VALUE_EQ_ITERATION), VALUE_EQ_255), stream(0, 1, 2, 3, 255), enc(), true }, { "[0, 1, 2, 3] repn=4(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, { "[255, 0, 1, 2, 3, 255] def(255), while<3(CURRENT_ITERATION), def (255)", seq(VALUE_EQ_255, whl(VALUE_EQ_ITERATION, not(eq(con(3)))), VALUE_EQ_255), stream(255, 0, 1, 2, 3, 255), enc(), true }, - { "[0, 0, 1, 2, 1, 0, 1, 2] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3))), con(2)), stream(0, 0, 1, 2, 1, 0, 1, 2), enc(), true }, + { "[0, 0, 1, 2, 255, 1, 0, 1, 2, 255] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3)), VALUE_EQ_255), con(2)), stream(0, 0, 1, 2, 255, 1, 0, 1, 2, 255), enc(), true }, { "[0, 1] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 1), enc(), false }, { "[0] CURRENT_ITERATION", VALUE_EQ_ITERATION, stream(0), enc(), false }, { "[0, 1] seq(!CURRENT_ITERATION, ...)", seq(VALUE_NOT_EQ_ITERATION, VALUE_NOT_EQ_ITERATION), stream(0, 1), enc(), true }, - { "[0] !CURRENT_ITERATION", VALUE_NOT_EQ_ITERATION, stream(0), enc(), true } + { "[0] !CURRENT_ITERATION", VALUE_NOT_EQ_ITERATION, stream(0), enc(), true }, + { "[0 | 0, 1 | 0, 0, 2 | 0, 0, 0, 3] rep(CURRENT_ITERATION)", rep(def("value", add(CURRENT_ITERATION, con(1)), eqNum(CURRENT_ITERATION))), stream(0, 0, 1, 0, 0, 2, 0, 0, 0, 3), enc(), true }, + { "[1, 1, 0, 1 | 0 | 2 | 3] repn=4(size), rep(def(any, sizeRef(CURRENT_INDEX)))", seq(repn(def("size", 1), con(4)), rep(def("value", nth(ref("size"), CURRENT_ITERATION)))), stream(1, 1, 0, 1, 0, 2, 3), enc(), true }, }); } From ebfcf71df55b1257cbcdd80447535a0740b803c6 Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Tue, 21 Aug 2018 00:46:16 +0200 Subject: [PATCH 17/26] #236: Removed redundant import. --- core/src/main/java/io/parsingdata/metal/data/ParseState.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/main/java/io/parsingdata/metal/data/ParseState.java b/core/src/main/java/io/parsingdata/metal/data/ParseState.java index e28207b4..00626bc6 100644 --- a/core/src/main/java/io/parsingdata/metal/data/ParseState.java +++ b/core/src/main/java/io/parsingdata/metal/data/ParseState.java @@ -17,7 +17,6 @@ package io.parsingdata.metal.data; import static java.math.BigInteger.ONE; -import static java.math.BigInteger.TEN; import static java.math.BigInteger.ZERO; import static io.parsingdata.metal.Util.checkNotNegative; From 6b728491f2b1f2ebc0012e2f72c9161b3fa357aa Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Fri, 24 Aug 2018 23:48:15 +0200 Subject: [PATCH 18/26] #236: Processed review comments of @rdvdijk. --- .gitignore | 1 - .../io/parsingdata/metal/token/IterableToken.java | 15 +++++++++++++++ .../value/reference/CurrentIterationTest.java | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9fb3b0a9..e87b1635 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ # Maven target/ -out/ # Eclipse .settings/ diff --git a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java index d2d0d207..45b7b774 100644 --- a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java +++ b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java @@ -1,3 +1,18 @@ +/* + * Copyright 2013-2018 Netherlands Forensic Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package io.parsingdata.metal.token; import io.parsingdata.metal.Trampoline; diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index 1541c4fa..c65533a8 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -51,7 +51,7 @@ public static Collection data() { return Arrays.asList(new Object[][] { { "[0, 1, 2, 3, 255] rep(CURRENT_ITERATION), def(255)", seq(rep(VALUE_EQ_ITERATION), VALUE_EQ_255), stream(0, 1, 2, 3, 255), enc(), true }, { "[0, 1, 2, 3] repn=4(CURRENT_ITERATION)", repn(VALUE_EQ_ITERATION, con(4)), stream(0, 1, 2, 3), enc(), true }, - { "[255, 0, 1, 2, 3, 255] def(255), while<3(CURRENT_ITERATION), def (255)", seq(VALUE_EQ_255, whl(VALUE_EQ_ITERATION, not(eq(con(3)))), VALUE_EQ_255), stream(255, 0, 1, 2, 3, 255), enc(), true }, + { "[255, 0, 1, 2, 3, 255] def(255), while!=3(CURRENT_ITERATION), def (255)", seq(VALUE_EQ_255, whl(VALUE_EQ_ITERATION, not(eq(con(3)))), VALUE_EQ_255), stream(255, 0, 1, 2, 3, 255), enc(), true }, { "[0, 0, 1, 2, 255, 1, 0, 1, 2, 255] repn=2(CURRENT_ITERATION, repn=3(CURRENT_ITERATION))", repn(seq(VALUE_EQ_ITERATION, repn(VALUE_EQ_ITERATION, con(3)), VALUE_EQ_255), con(2)), stream(0, 0, 1, 2, 255, 1, 0, 1, 2, 255), enc(), true }, { "[0, 1] seq(CURRENT_ITERATION, ...)", seq(VALUE_EQ_ITERATION, VALUE_EQ_ITERATION), stream(0, 1), enc(), false }, { "[0] CURRENT_ITERATION", VALUE_EQ_ITERATION, stream(0), enc(), false }, From fe71ac29b9ed838fc25edb5d8a32f536647cb8d8 Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Sat, 25 Aug 2018 00:00:12 +0200 Subject: [PATCH 19/26] #236: Included ToString test for ParseState with iterations. --- core/src/test/java/io/parsingdata/metal/ToStringTest.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/src/test/java/io/parsingdata/metal/ToStringTest.java b/core/src/test/java/io/parsingdata/metal/ToStringTest.java index 9239bc93..d82bcaaa 100644 --- a/core/src/test/java/io/parsingdata/metal/ToStringTest.java +++ b/core/src/test/java/io/parsingdata/metal/ToStringTest.java @@ -75,6 +75,7 @@ import java.nio.charset.StandardCharsets; import java.util.Optional; +import io.parsingdata.metal.token.Rep; import org.junit.Before; import org.junit.Test; @@ -166,6 +167,9 @@ public void data() { final ParseState parseState = stream(1, 2); final String parseStateString = "ParseState(source:ByteStreamSource(InMemoryByteStream(2));offset:0;order:pg(EMPTY))"; assertEquals(parseStateString, parseState.toString()); + final ParseState parseStateWithIterations = parseState.addBranch(rep(def("a",1))).iter(); + final String parseStateWithIterationsString = "ParseState(source:ByteStreamSource(InMemoryByteStream(2));offset:0;order:pg(pg(terminator:Rep),pg(EMPTY),true);iterations:>1)"; + assertEquals(parseStateWithIterationsString, parseStateWithIterations.toString()); final Optional result = Optional.of(parseState); assertEquals("Optional[" + parseState + "]", result.toString()); final ParseValue pv1 = new ParseValue("name", NONE, createFromBytes(new byte[]{1, 2}), enc()); From 8f9d2c39f5f0824c38cc082752fde7c4965b5da2 Mon Sep 17 00:00:00 2001 From: Marja van Aken Date: Mon, 3 Sep 2018 10:26:23 +0200 Subject: [PATCH 20/26] #236: Processed review comments from @rdvdijk @ccreeten. --- .../io/parsingdata/metal/data/ParseState.java | 22 +++++++++---------- .../value/reference/CurrentIteration.java | 5 +++-- .../metal/token/IterableToken.java | 11 +++++----- .../java/io/parsingdata/metal/token/RepN.java | 2 +- .../io/parsingdata/metal/ToStringTest.java | 3 +-- 5 files changed, 21 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/data/ParseState.java b/core/src/main/java/io/parsingdata/metal/data/ParseState.java index 00626bc6..d915a7f5 100644 --- a/core/src/main/java/io/parsingdata/metal/data/ParseState.java +++ b/core/src/main/java/io/parsingdata/metal/data/ParseState.java @@ -31,19 +31,20 @@ import io.parsingdata.metal.encoding.Encoding; import io.parsingdata.metal.expression.value.ValueExpression; import io.parsingdata.metal.token.Token; +import javafx.util.Pair; public class ParseState { public final ParseGraph order; public final BigInteger offset; public final Source source; - public final ImmutableList iterations; + public final ImmutableList> iterations; - public ParseState(final ParseGraph order, final Source source, final BigInteger offset, final ImmutableList iterations) { + public ParseState(final ParseGraph order, final Source source, final BigInteger offset, final ImmutableList> iterations) { this.order = checkNotNull(order, "order"); this.source = checkNotNull(source, "source"); this.offset = checkNotNegative(offset, "offset"); - this.iterations = checkNotNull(iterations, "iteration"); + this.iterations = checkNotNull(iterations, "iterations"); } public static ParseState createFromByteStream(final ByteStream input, final BigInteger offset) { @@ -55,17 +56,14 @@ public static ParseState createFromByteStream(final ByteStream input) { } public ParseState addBranch(final Token token) { - if (token.isIterable()) { - return new ParseState(order.addBranch(token), source, offset, iterations.add(ZERO)); - } - return new ParseState(order.addBranch(token), source, offset, iterations); + return new ParseState(order.addBranch(token), source, offset, token.isIterable() ? iterations.add(new Pair<>(token, ZERO)) : iterations); } public ParseState closeBranch(final Token token) { - if (token.isIterable()) { - return new ParseState(order.closeBranch(), source, offset, iterations.tail); + if (token.isIterable() && !iterations.head.getKey().equals(token)) { + throw new IllegalStateException(String.format("Cannot close branch for iterable token %s. Current iteration state is for token %s.", token.name, iterations.head.getKey().name)); } - return new ParseState(order.closeBranch(), source, offset, iterations); + return new ParseState(order.closeBranch(), source, offset, token.isIterable() ? iterations.tail : iterations); } public ParseState add(final ParseValue parseValue) { @@ -76,8 +74,8 @@ public ParseState add(final ParseReference parseReference) { return new ParseState(order.add(parseReference), source, offset, iterations); } - public ParseState iter() { - return new ParseState(order, source, offset, iterations.tail.add(iterations.head.add(ONE))); + public ParseState iterate() { + return new ParseState(order, source, offset, iterations.tail.add(new Pair<>(iterations.head.getKey(), iterations.head.getValue().add(ONE)))); } public Optional seek(final BigInteger newOffset) { diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index ac7ffad3..a65c953d 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -32,7 +32,8 @@ /** * A {@link ValueExpression} that represents the 0-based current iteration in an - * iterable {@link Token} (e.g. when inside a {@link Rep} or {@link RepN}). + * iterable {@link Token} (when {@link Token#isIterable()} returns true, e.g. when + * inside a {@link Rep} or {@link RepN}) */ public class CurrentIteration implements ValueExpression { @@ -41,7 +42,7 @@ public ImmutableList> eval(final ParseState parseState, final En if (parseState.iterations.head == null) { return ImmutableList.create(Optional.empty()); } - return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head, new Encoding()))); + return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head.getValue(), new Encoding()))); } @Override diff --git a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java index 45b7b774..5b5cdc51 100644 --- a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java +++ b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java @@ -23,6 +23,7 @@ import java.util.Objects; import java.util.Optional; import java.util.function.Function; +import java.util.function.Predicate; import static io.parsingdata.metal.Trampoline.complete; import static io.parsingdata.metal.Trampoline.intermediate; @@ -33,12 +34,12 @@ public abstract class IterableToken extends Token { public final Token token; - IterableToken(String name, final Token token, Encoding encoding) { + IterableToken(final String name, final Token token, final Encoding encoding) { super(name, encoding); this.token = checkNotNull(token, "token"); } - protected final Optional parse(final Environment environment, final Function stopCondition, final Function> ifIterationFails) { + protected final Optional parse(final Environment environment, final Predicate stopCondition, final Function> ifIterationFails) { return iterate(environment.addBranch(this), stopCondition, ifIterationFails).computeResult(); } @@ -50,13 +51,13 @@ protected final Optional parse(final Environment environment, final * @param ifIterationFails a function to determine how to handle a failed parse * @return a trampolined {@code Optional} */ - private Trampoline> iterate(final Environment environment, final Function stopCondition, final Function> ifIterationFails) { - if (stopCondition.apply(environment)) { + private Trampoline> iterate(final Environment environment, final Predicate stopCondition, final Function> ifIterationFails) { + if (stopCondition.test(environment)) { return complete(() -> success(environment.parseState.closeBranch(this))); } return token .parse(environment) - .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState.iter()), stopCondition, ifIterationFails))) + .map(nextParseState -> intermediate(() -> iterate(environment.withParseState(nextParseState.iterate()), stopCondition, ifIterationFails))) .orElseGet(() -> complete(() -> ifIterationFails.apply(environment))); } diff --git a/core/src/main/java/io/parsingdata/metal/token/RepN.java b/core/src/main/java/io/parsingdata/metal/token/RepN.java index 69da009f..13d7efcd 100644 --- a/core/src/main/java/io/parsingdata/metal/token/RepN.java +++ b/core/src/main/java/io/parsingdata/metal/token/RepN.java @@ -58,7 +58,7 @@ protected Optional parseImpl(final Environment environment) { return failure(); } final BigInteger count = counts.head.get().asNumeric(); - return parse(environment, env -> env.parseState.iterations.head.compareTo(count) >= 0, env -> failure()); + return parse(environment, env -> env.parseState.iterations.head.getValue().compareTo(count) >= 0, env -> failure()); } @Override diff --git a/core/src/test/java/io/parsingdata/metal/ToStringTest.java b/core/src/test/java/io/parsingdata/metal/ToStringTest.java index d82bcaaa..6cd075df 100644 --- a/core/src/test/java/io/parsingdata/metal/ToStringTest.java +++ b/core/src/test/java/io/parsingdata/metal/ToStringTest.java @@ -75,7 +75,6 @@ import java.nio.charset.StandardCharsets; import java.util.Optional; -import io.parsingdata.metal.token.Rep; import org.junit.Before; import org.junit.Test; @@ -167,7 +166,7 @@ public void data() { final ParseState parseState = stream(1, 2); final String parseStateString = "ParseState(source:ByteStreamSource(InMemoryByteStream(2));offset:0;order:pg(EMPTY))"; assertEquals(parseStateString, parseState.toString()); - final ParseState parseStateWithIterations = parseState.addBranch(rep(def("a",1))).iter(); + final ParseState parseStateWithIterations = parseState.addBranch(rep(def("a",1))).iterate(); final String parseStateWithIterationsString = "ParseState(source:ByteStreamSource(InMemoryByteStream(2));offset:0;order:pg(pg(terminator:Rep),pg(EMPTY),true);iterations:>1)"; assertEquals(parseStateWithIterationsString, parseStateWithIterations.toString()); final Optional result = Optional.of(parseState); From 97a66c565194c1faee2766e4d98e156fc1c20979 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Mon, 3 Sep 2018 12:48:36 +0200 Subject: [PATCH 21/26] #236: Fixed a failing test. --- core/src/test/java/io/parsingdata/metal/ToStringTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/io/parsingdata/metal/ToStringTest.java b/core/src/test/java/io/parsingdata/metal/ToStringTest.java index 6cd075df..ae8bfbd9 100644 --- a/core/src/test/java/io/parsingdata/metal/ToStringTest.java +++ b/core/src/test/java/io/parsingdata/metal/ToStringTest.java @@ -167,7 +167,7 @@ public void data() { final String parseStateString = "ParseState(source:ByteStreamSource(InMemoryByteStream(2));offset:0;order:pg(EMPTY))"; assertEquals(parseStateString, parseState.toString()); final ParseState parseStateWithIterations = parseState.addBranch(rep(def("a",1))).iterate(); - final String parseStateWithIterationsString = "ParseState(source:ByteStreamSource(InMemoryByteStream(2));offset:0;order:pg(pg(terminator:Rep),pg(EMPTY),true);iterations:>1)"; + final String parseStateWithIterationsString = "ParseState(source:ByteStreamSource(InMemoryByteStream(2));offset:0;order:pg(pg(terminator:Rep),pg(EMPTY),true);iterations:>Rep(Def(a,Const(0x01)))=1)"; assertEquals(parseStateWithIterationsString, parseStateWithIterations.toString()); final Optional result = Optional.of(parseState); assertEquals("Optional[" + parseState + "]", result.toString()); From 5442e7a9749abd6592aa4037aa696b804423dd5e Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Mon, 10 Sep 2018 14:45:37 +0200 Subject: [PATCH 22/26] #236: Added test to repair test coverage. --- .../metal/data/ParseStateTest.java | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 core/src/test/java/io/parsingdata/metal/data/ParseStateTest.java diff --git a/core/src/test/java/io/parsingdata/metal/data/ParseStateTest.java b/core/src/test/java/io/parsingdata/metal/data/ParseStateTest.java new file mode 100644 index 00000000..d5e2edf4 --- /dev/null +++ b/core/src/test/java/io/parsingdata/metal/data/ParseStateTest.java @@ -0,0 +1,43 @@ +/* + * Copyright 2013-2016 Netherlands Forensic Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.parsingdata.metal.data; + +import static io.parsingdata.metal.Shorthand.rep; +import static io.parsingdata.metal.util.ParseStateFactory.stream; +import static io.parsingdata.metal.util.TokenDefinitions.any; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import io.parsingdata.metal.token.Token; + +public class ParseStateTest { + + @Rule + public final ExpectedException thrown = ExpectedException.none(); + + @Test + public void closeParseStateWithWrongToken() { + thrown.expect(IllegalStateException.class); + thrown.expectMessage("Cannot close branch for iterable token closeName. Current iteration state is for token openName."); + final Token open = rep("openName", any("a")); + final Token close = rep("closeName", any("a")); + stream().addBranch(open).closeBranch(close); + } + +} \ No newline at end of file From 93716c39e9b1bf0c426d84dde0af97cc636b54ee Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Sat, 15 Sep 2018 14:58:49 +0200 Subject: [PATCH 23/26] 236: Improved CurrentIterationTest. --- .../metal/expression/value/reference/CurrentIterationTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index c65533a8..8583e24a 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -58,7 +58,7 @@ public static Collection data() { { "[0, 1] seq(!CURRENT_ITERATION, ...)", seq(VALUE_NOT_EQ_ITERATION, VALUE_NOT_EQ_ITERATION), stream(0, 1), enc(), true }, { "[0] !CURRENT_ITERATION", VALUE_NOT_EQ_ITERATION, stream(0), enc(), true }, { "[0 | 0, 1 | 0, 0, 2 | 0, 0, 0, 3] rep(CURRENT_ITERATION)", rep(def("value", add(CURRENT_ITERATION, con(1)), eqNum(CURRENT_ITERATION))), stream(0, 0, 1, 0, 0, 2, 0, 0, 0, 3), enc(), true }, - { "[1, 1, 0, 1 | 0 | 2 | 3] repn=4(size), rep(def(any, sizeRef(CURRENT_INDEX)))", seq(repn(def("size", 1), con(4)), rep(def("value", nth(ref("size"), CURRENT_ITERATION)))), stream(1, 1, 0, 1, 0, 2, 3), enc(), true }, + { "[1, 1, 0, 1 | 0 | 1 | 3] repn=4(size), rep(def(any, sizeRef(CURRENT_INDEX)))", seq(repn(def("size", 1), con(4)), repn(def("value", nth(ref("size"), CURRENT_ITERATION)), con(4))), stream(1, 1, 0, 1, 0, 1, 3), enc(), true }, }); } From fd803d82c23bd52f301270448c60f060ea48f4ea Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Sat, 15 Sep 2018 15:14:53 +0200 Subject: [PATCH 24/26] 236: Improved CurrentIterationTest to actually verify zero-sized Def and CURRENT_ITERATION behaviour inside an IterableToken. --- .../metal/expression/value/reference/CurrentIterationTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java index 8583e24a..722ed24d 100644 --- a/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java +++ b/core/src/test/java/io/parsingdata/metal/expression/value/reference/CurrentIterationTest.java @@ -28,6 +28,7 @@ import static io.parsingdata.metal.Shorthand.rep; import static io.parsingdata.metal.Shorthand.repn; import static io.parsingdata.metal.Shorthand.seq; +import static io.parsingdata.metal.Shorthand.when; import static io.parsingdata.metal.Shorthand.whl; import static io.parsingdata.metal.util.EncodingFactory.enc; import static io.parsingdata.metal.util.ParseStateFactory.stream; @@ -58,7 +59,7 @@ public static Collection data() { { "[0, 1] seq(!CURRENT_ITERATION, ...)", seq(VALUE_NOT_EQ_ITERATION, VALUE_NOT_EQ_ITERATION), stream(0, 1), enc(), true }, { "[0] !CURRENT_ITERATION", VALUE_NOT_EQ_ITERATION, stream(0), enc(), true }, { "[0 | 0, 1 | 0, 0, 2 | 0, 0, 0, 3] rep(CURRENT_ITERATION)", rep(def("value", add(CURRENT_ITERATION, con(1)), eqNum(CURRENT_ITERATION))), stream(0, 0, 1, 0, 0, 2, 0, 0, 0, 3), enc(), true }, - { "[1, 1, 0, 1 | 0 | 1 | 3] repn=4(size), rep(def(any, sizeRef(CURRENT_INDEX)))", seq(repn(def("size", 1), con(4)), repn(def("value", nth(ref("size"), CURRENT_ITERATION)), con(4))), stream(1, 1, 0, 1, 0, 1, 3), enc(), true }, + { "[1, 1, 0, 1 | 0 | 1 | 3] repn=4(def(size)), repn=4(if(sizeRef(CURRENT_ITERATION) != 0, def(CURRENT_ITERATION)))", seq(repn(def("size", 1), con(4)), repn(when(def("value", con(1), eq(CURRENT_ITERATION)), not(eq(nth(ref("size"), CURRENT_ITERATION), con(0)))), con(4))), stream(1, 1, 0, 1, 0, 1, 3), enc(), true }, }); } From c63f84c62eb4ba990dd099d729343be3905ea761 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Sun, 16 Sep 2018 14:23:45 +0200 Subject: [PATCH 25/26] #236: Replaced null-check with isEmpty()-call, added newline, improved javadoc. Based on review comments from @rdvdijk and @ccreeten. --- .../metal/expression/value/reference/CurrentIteration.java | 5 +++-- .../main/java/io/parsingdata/metal/token/IterableToken.java | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index a65c953d..65eb2548 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -29,17 +29,18 @@ import io.parsingdata.metal.token.Rep; import io.parsingdata.metal.token.RepN; import io.parsingdata.metal.token.Token; +import io.parsingdata.metal.token.While; /** * A {@link ValueExpression} that represents the 0-based current iteration in an * iterable {@link Token} (when {@link Token#isIterable()} returns true, e.g. when - * inside a {@link Rep} or {@link RepN}) + * inside a {@link Rep}, {@link RepN}) or {@link While}). */ public class CurrentIteration implements ValueExpression { @Override public ImmutableList> eval(final ParseState parseState, final Encoding encoding) { - if (parseState.iterations.head == null) { + if (parseState.iterations.isEmpty()) { return ImmutableList.create(Optional.empty()); } return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head.getValue(), new Encoding()))); diff --git a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java index 5b5cdc51..f7f410bf 100644 --- a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java +++ b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package io.parsingdata.metal.token; import io.parsingdata.metal.Trampoline; From 840c354b651b979e7a06260546f2466ef4a3a294 Mon Sep 17 00:00:00 2001 From: Jeroen van den Bos Date: Sun, 16 Sep 2018 17:15:12 +0200 Subject: [PATCH 26/26] #236: Removed dependency on the javafx Pair class by adding trivial ImmutablePair class. --- .../java/io/parsingdata/metal/Shorthand.java | 4 +- .../parsingdata/metal/data/ImmutablePair.java | 52 +++++++++++++++++++ .../io/parsingdata/metal/data/ParseState.java | 13 +++-- .../value/reference/CurrentIteration.java | 2 +- .../metal/token/IterableToken.java | 16 +++--- .../java/io/parsingdata/metal/token/RepN.java | 2 +- .../parsingdata/metal/AutoEqualityTest.java | 2 +- .../io/parsingdata/metal/EqualityTest.java | 17 +++++- 8 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 core/src/main/java/io/parsingdata/metal/data/ImmutablePair.java diff --git a/core/src/main/java/io/parsingdata/metal/Shorthand.java b/core/src/main/java/io/parsingdata/metal/Shorthand.java index 2b2fb7b2..6f29580a 100644 --- a/core/src/main/java/io/parsingdata/metal/Shorthand.java +++ b/core/src/main/java/io/parsingdata/metal/Shorthand.java @@ -44,12 +44,11 @@ import io.parsingdata.metal.expression.value.Cat; import io.parsingdata.metal.expression.value.Const; import io.parsingdata.metal.expression.value.ConstantFactory; -import io.parsingdata.metal.expression.value.reference.CurrentIteration; import io.parsingdata.metal.expression.value.Elvis; import io.parsingdata.metal.expression.value.Expand; +import io.parsingdata.metal.expression.value.FoldCat; import io.parsingdata.metal.expression.value.FoldLeft; import io.parsingdata.metal.expression.value.FoldRight; -import io.parsingdata.metal.expression.value.FoldCat; import io.parsingdata.metal.expression.value.Reverse; import io.parsingdata.metal.expression.value.UnaryValueExpression; import io.parsingdata.metal.expression.value.Value; @@ -63,6 +62,7 @@ import io.parsingdata.metal.expression.value.bitwise.ShiftLeft; import io.parsingdata.metal.expression.value.bitwise.ShiftRight; import io.parsingdata.metal.expression.value.reference.Count; +import io.parsingdata.metal.expression.value.reference.CurrentIteration; import io.parsingdata.metal.expression.value.reference.CurrentOffset; import io.parsingdata.metal.expression.value.reference.First; import io.parsingdata.metal.expression.value.reference.Last; diff --git a/core/src/main/java/io/parsingdata/metal/data/ImmutablePair.java b/core/src/main/java/io/parsingdata/metal/data/ImmutablePair.java new file mode 100644 index 00000000..486bb17a --- /dev/null +++ b/core/src/main/java/io/parsingdata/metal/data/ImmutablePair.java @@ -0,0 +1,52 @@ +/* + * Copyright 2013-2016 Netherlands Forensic Institute + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.parsingdata.metal.data; + +import static io.parsingdata.metal.Util.checkNotNull; + +import java.util.Objects; + +import io.parsingdata.metal.Util; + +public class ImmutablePair { + + public final L left; + public final R right; + + public ImmutablePair(final L left, final R right) { + this.left = checkNotNull(left, "left"); + this.right = checkNotNull(right, "right"); + } + + @Override + public String toString() { + return left + "=" + right; + } + + @Override + public boolean equals(final Object obj) { + return Util.notNullAndSameClass(this, obj) + && Objects.equals(left, ((ImmutablePair)obj).left) + && Objects.equals(right, ((ImmutablePair)obj).right); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), left, right); + } + +} diff --git a/core/src/main/java/io/parsingdata/metal/data/ParseState.java b/core/src/main/java/io/parsingdata/metal/data/ParseState.java index d915a7f5..e394b4d4 100644 --- a/core/src/main/java/io/parsingdata/metal/data/ParseState.java +++ b/core/src/main/java/io/parsingdata/metal/data/ParseState.java @@ -31,16 +31,15 @@ import io.parsingdata.metal.encoding.Encoding; import io.parsingdata.metal.expression.value.ValueExpression; import io.parsingdata.metal.token.Token; -import javafx.util.Pair; public class ParseState { public final ParseGraph order; public final BigInteger offset; public final Source source; - public final ImmutableList> iterations; + public final ImmutableList> iterations; - public ParseState(final ParseGraph order, final Source source, final BigInteger offset, final ImmutableList> iterations) { + public ParseState(final ParseGraph order, final Source source, final BigInteger offset, final ImmutableList> iterations) { this.order = checkNotNull(order, "order"); this.source = checkNotNull(source, "source"); this.offset = checkNotNegative(offset, "offset"); @@ -56,12 +55,12 @@ public static ParseState createFromByteStream(final ByteStream input) { } public ParseState addBranch(final Token token) { - return new ParseState(order.addBranch(token), source, offset, token.isIterable() ? iterations.add(new Pair<>(token, ZERO)) : iterations); + return new ParseState(order.addBranch(token), source, offset, token.isIterable() ? iterations.add(new ImmutablePair<>(token, ZERO)) : iterations); } public ParseState closeBranch(final Token token) { - if (token.isIterable() && !iterations.head.getKey().equals(token)) { - throw new IllegalStateException(String.format("Cannot close branch for iterable token %s. Current iteration state is for token %s.", token.name, iterations.head.getKey().name)); + if (token.isIterable() && !iterations.head.left.equals(token)) { + throw new IllegalStateException(String.format("Cannot close branch for iterable token %s. Current iteration state is for token %s.", token.name, iterations.head.left.name)); } return new ParseState(order.closeBranch(), source, offset, token.isIterable() ? iterations.tail : iterations); } @@ -75,7 +74,7 @@ public ParseState add(final ParseReference parseReference) { } public ParseState iterate() { - return new ParseState(order, source, offset, iterations.tail.add(new Pair<>(iterations.head.getKey(), iterations.head.getValue().add(ONE)))); + return new ParseState(order, source, offset, iterations.tail.add(new ImmutablePair<>(iterations.head.left, iterations.head.right.add(ONE)))); } public Optional seek(final BigInteger newOffset) { diff --git a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java index 65eb2548..543f159d 100644 --- a/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java +++ b/core/src/main/java/io/parsingdata/metal/expression/value/reference/CurrentIteration.java @@ -43,7 +43,7 @@ public ImmutableList> eval(final ParseState parseState, final En if (parseState.iterations.isEmpty()) { return ImmutableList.create(Optional.empty()); } - return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head.getValue(), new Encoding()))); + return ImmutableList.create(Optional.of(createFromNumeric(parseState.iterations.head.right, new Encoding()))); } @Override diff --git a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java index f7f410bf..0ea8c014 100644 --- a/core/src/main/java/io/parsingdata/metal/token/IterableToken.java +++ b/core/src/main/java/io/parsingdata/metal/token/IterableToken.java @@ -16,20 +16,20 @@ package io.parsingdata.metal.token; -import io.parsingdata.metal.Trampoline; -import io.parsingdata.metal.data.Environment; -import io.parsingdata.metal.data.ParseState; -import io.parsingdata.metal.encoding.Encoding; +import static io.parsingdata.metal.Trampoline.complete; +import static io.parsingdata.metal.Trampoline.intermediate; +import static io.parsingdata.metal.Util.checkNotNull; +import static io.parsingdata.metal.Util.success; import java.util.Objects; import java.util.Optional; import java.util.function.Function; import java.util.function.Predicate; -import static io.parsingdata.metal.Trampoline.complete; -import static io.parsingdata.metal.Trampoline.intermediate; -import static io.parsingdata.metal.Util.checkNotNull; -import static io.parsingdata.metal.Util.success; +import io.parsingdata.metal.Trampoline; +import io.parsingdata.metal.data.Environment; +import io.parsingdata.metal.data.ParseState; +import io.parsingdata.metal.encoding.Encoding; public abstract class IterableToken extends Token { diff --git a/core/src/main/java/io/parsingdata/metal/token/RepN.java b/core/src/main/java/io/parsingdata/metal/token/RepN.java index 13d7efcd..3a9b61fb 100644 --- a/core/src/main/java/io/parsingdata/metal/token/RepN.java +++ b/core/src/main/java/io/parsingdata/metal/token/RepN.java @@ -58,7 +58,7 @@ protected Optional parseImpl(final Environment environment) { return failure(); } final BigInteger count = counts.head.get().asNumeric(); - return parse(environment, env -> env.parseState.iterations.head.getValue().compareTo(count) >= 0, env -> failure()); + return parse(environment, env -> env.parseState.iterations.head.right.compareTo(count) >= 0, env -> failure()); } @Override diff --git a/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java b/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java index fbd70dcd..0e0a377e 100644 --- a/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java +++ b/core/src/test/java/io/parsingdata/metal/AutoEqualityTest.java @@ -79,11 +79,11 @@ import io.parsingdata.metal.expression.comparison.LtNum; import io.parsingdata.metal.expression.value.Bytes; import io.parsingdata.metal.expression.value.Cat; -import io.parsingdata.metal.expression.value.FoldCat; import io.parsingdata.metal.expression.value.Const; import io.parsingdata.metal.expression.value.ConstantFactory; import io.parsingdata.metal.expression.value.Elvis; import io.parsingdata.metal.expression.value.Expand; +import io.parsingdata.metal.expression.value.FoldCat; import io.parsingdata.metal.expression.value.FoldLeft; import io.parsingdata.metal.expression.value.FoldRight; import io.parsingdata.metal.expression.value.Reverse; diff --git a/core/src/test/java/io/parsingdata/metal/EqualityTest.java b/core/src/test/java/io/parsingdata/metal/EqualityTest.java index e7d4b90a..faf7aaa7 100644 --- a/core/src/test/java/io/parsingdata/metal/EqualityTest.java +++ b/core/src/test/java/io/parsingdata/metal/EqualityTest.java @@ -56,6 +56,7 @@ import io.parsingdata.metal.data.ConstantSource; import io.parsingdata.metal.data.ImmutableList; +import io.parsingdata.metal.data.ImmutablePair; import io.parsingdata.metal.data.ParseGraph; import io.parsingdata.metal.data.ParseState; import io.parsingdata.metal.data.ParseValue; @@ -155,7 +156,7 @@ public void multiConstructorTypes() { @Test public void immutableList() { - final ImmutableList object = ImmutableList.create("a"); + final ImmutableList object = ImmutableList.create("a"); assertFalse(object.equals(null)); assertTrue(object.equals(ImmutableList.create("a"))); assertTrue(object.equals(new ImmutableList<>().add("a"))); @@ -166,6 +167,20 @@ public void immutableList() { assertFalse(object.add("b").add("c").equals(ImmutableList.create("a").add("c").add("c"))); } + @Test + public void immutablePair() { + final ImmutablePair object = new ImmutablePair("a", ONE); + assertFalse(object.equals(null)); + assertTrue(object.equals(new ImmutablePair("a", ONE))); + assertEquals(object.hashCode(), new ImmutablePair("a", ONE).hashCode()); + assertFalse(object.equals("a")); + assertNotEquals(object.hashCode(), "a".hashCode()); + assertFalse(object.equals(new ImmutablePair("b", ONE))); + assertNotEquals(object.hashCode(), new ImmutablePair("b", ONE).hashCode()); + assertFalse(object.equals(new ImmutablePair("a", ZERO))); + assertNotEquals(object.hashCode(), new ImmutablePair("a", ZERO).hashCode()); + } + @Test public void parseGraph() { final ParseValue value = new ParseValue("a", any("a"), createFromBytes(new byte[]{1, 2}), enc());