-
Notifications
You must be signed in to change notification settings - Fork 62
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Adding optional Configuration to Machine that can be used to enable a… #125
Changes from 1 commit
8251544
32d8015
e94c1a9
3732a4f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
package software.amazon.event.ruler; | ||
|
||
/** | ||
* Configuration for a Machine. | ||
*/ | ||
public class Configuration { | ||
|
||
/** | ||
* Normally, NameStates are re-used for a given key subsequence and pattern if this key subsequence and pattern have | ||
* been previously added, or if the current rule has already added a pattern for the given key subsequence. Hence, | ||
* by default, NameState re-use is opportunistic. But by setting this flag to true, NameState re-use will be forced | ||
* for a key subsequence. This means that the first pattern being added for a key subsequence for a rule will re-use | ||
* a NameState if that key subsequence has been added before. Meaning each key subsequence has a single NameState. | ||
* This improves memory utilization exponentially in some cases but does lead to more sub-rules being stored in | ||
* individual NameStates, which Ruler sometimes iterates over, which can cause a modest runtime performance | ||
* regression. | ||
*/ | ||
private final boolean additionalNameStateReuse; | ||
|
||
private Configuration(boolean additionalNameStateReuse) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Do you see this being used for configuration beyond Machine? If not, we can be explicit by calling this class MachingConfiguration There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It's possible that some configuration will not be related directly to Machine, but interface-wise, it will always be plumbed in via the Machine. So the name change is reasonable. Although to be fully correct, it should be GenericMachineConfiguration. I will perform this rename, although personally, I don't think the length and clunkiness is worth the explicitness. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. FWIW, it hadn't cross my mind that we'd implement anything other than |
||
this.additionalNameStateReuse = additionalNameStateReuse; | ||
} | ||
|
||
public boolean isAdditionalNameStateReuse() { | ||
return additionalNameStateReuse; | ||
} | ||
|
||
public static class Builder { | ||
|
||
private boolean additionalNameStateReuse = false; | ||
|
||
public Builder withAdditionalNameStateReuse(boolean additionalNameStateReuse) { | ||
this.additionalNameStateReuse = additionalNameStateReuse; | ||
return this; | ||
} | ||
|
||
public Configuration build() { | ||
return new Configuration(additionalNameStateReuse); | ||
} | ||
} | ||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
package software.amazon.event.ruler; | ||
|
||
import org.junit.Test; | ||
|
||
import static org.junit.Assert.assertFalse; | ||
import static org.junit.Assert.assertTrue; | ||
|
||
public class ConfigurationTest { | ||
|
||
@Test | ||
public void testAdditionalNameStateReuseTrue() { | ||
assertTrue(new Configuration.Builder().withAdditionalNameStateReuse(true).build().isAdditionalNameStateReuse()); | ||
} | ||
|
||
@Test | ||
public void testAdditionalNameStateReuseFalse() { | ||
assertFalse(new Configuration.Builder().withAdditionalNameStateReuse(false).build().isAdditionalNameStateReuse()); | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2621,4 +2621,46 @@ public void testLargeArrayRulesVsOR() throws Exception { | |
"}"); | ||
assertEquals(608, machine.approximateObjectCount(10000)); | ||
} | ||
|
||
@Test | ||
public void testApproximateObjectCountEachKeyHasThreePatternsAddedOneAtATime() throws Exception { | ||
Machine machine = new Machine(); | ||
testApproximateObjectCountEachKeyHasThreePatternsAddedOneAtATime(machine); | ||
assertEquals(72216, machine.approximateObjectCount(500000)); | ||
} | ||
|
||
@Test | ||
public void testApproximateObjectCountEachKeyHasThreePatternsAddedOneAtATimeWithAdditionalNameStateReuse() throws Exception { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should there be a new benchmark test to check for memory usage https://github.com/aws/event-ruler/blob/main/src/test/software/amazon/event/ruler/Benchmarks.java#L452 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ok added. Got these results over three runs: High NameState Reuse Memory Benchmark High NameState Reuse Memory Benchmark High NameState Reuse Memory Benchmark |
||
Machine machine = new Machine(new Configuration.Builder().withAdditionalNameStateReuse(true).build()); | ||
testApproximateObjectCountEachKeyHasThreePatternsAddedOneAtATime(machine); | ||
assertEquals(136, machine.approximateObjectCount(500000)); | ||
} | ||
|
||
private void testApproximateObjectCountEachKeyHasThreePatternsAddedOneAtATime(Machine machine) throws Exception { | ||
machine.addRule("0", "{\"key1\": [\"a\"]}"); | ||
machine.addRule("1", "{\"key1\": [\"b\"]}"); | ||
machine.addRule("2", "{\"key1\": [\"c\"]}"); | ||
machine.addRule("3", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\"]}"); | ||
machine.addRule("4", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"e\"]}"); | ||
machine.addRule("5", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"f\"]}"); | ||
machine.addRule("6", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\"]}"); | ||
machine.addRule("7", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"h\"]}"); | ||
machine.addRule("8", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"i\"]}"); | ||
machine.addRule("9", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\"]}"); | ||
machine.addRule("10", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"k\"]}"); | ||
machine.addRule("11", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"l\"]}"); | ||
machine.addRule("12", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\"]}"); | ||
machine.addRule("13", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"n\"]}"); | ||
machine.addRule("14", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"o\"]}"); | ||
machine.addRule("15", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\"]}"); | ||
machine.addRule("16", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"q\"]}"); | ||
machine.addRule("17", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"r\"]}"); | ||
machine.addRule("18", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\", \"q\", \"r\"], \"key7\": [\"s\"]}"); | ||
machine.addRule("19", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\", \"q\", \"r\"], \"key7\": [\"t\"]}"); | ||
machine.addRule("20", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\", \"q\", \"r\"], \"key7\": [\"u\"]}"); | ||
machine.addRule("21", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\", \"q\", \"r\"], \"key7\": [\"s\", \"t\", \"u\"], \"key8\": [\"v\"]}"); | ||
machine.addRule("22", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\", \"q\", \"r\"], \"key7\": [\"s\", \"t\", \"u\"], \"key8\": [\"w\"]}"); | ||
machine.addRule("23", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\", \"q\", \"r\"], \"key7\": [\"s\", \"t\", \"u\"], \"key8\": [\"x\"]}"); | ||
machine.addRule("24", "{\"key1\": [\"a\", \"b\", \"c\"], \"key2\": [\"d\", \"e\", \"f\"], \"key3\": [\"g\", \"h\", \"i\"], \"key4\": [\"j\", \"k\", \"l\"], \"key5\": [\"m\", \"n\", \"o\"], \"key6\": [\"p\", \"q\", \"r\"], \"key7\": [\"s\", \"t\", \"u\"], \"key8\": [\"v\", \"w\", \"x\"], \"key9\": [\"y\"]}"); | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just to note that this would need quite a bit of work before it went in the README. I found it really hard to understand. I suggest that the easiest way to make it comprehensible to mere mortals is to include a few examples.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Not sure it's "quite a bit of work", but I added to the readme with an example.