From fa020df5320ec2aaaeec29aa568352bca5022347 Mon Sep 17 00:00:00 2001 From: Stian Soiland-Reyes Date: Sat, 21 Mar 2015 02:51:28 +0000 Subject: [PATCH] JENA-901 Use Guava for cache, shadowed in jena-core this avoids adding a new module, so this is acceptable as a patch-version update. --- jena-core/pom.xml | 58 ++++++- .../reasoner/rulesys/impl/LPBRuleEngine.java | 116 +++++++++---- .../rulesys/impl/TestLPBRuleEngine.java | 163 ++++++++++++++++++ .../reasoner/rulesys/test/TestPackage.java | 4 + jena-parent/pom.xml | 5 + 5 files changed, 315 insertions(+), 31 deletions(-) create mode 100644 jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/impl/TestLPBRuleEngine.java diff --git a/jena-core/pom.xml b/jena-core/pom.xml index c6fbc5d7019..285edae432b 100644 --- a/jena-core/pom.xml +++ b/jena-core/pom.xml @@ -61,6 +61,22 @@ xercesImpl + + + com.google.guava + guava + + 18.0 + + + @@ -145,7 +161,7 @@ Apache Jena Apache Jena Core ${project.version} Licenced under the Apache License, Version 2.0 - com.hp.hpl.jena.shared.*:*.impl:com.hp.hpl.jena.assembler.assemblers:*.exceptions:*.regexptrees:com.hp.hpl.jena.mem:com.hp.hpl.jena.mem.*:com.hp.hpl.jena.n3:com.hp.hpl.jena.n3.*:com.hp.hpl.jena.rdf.arp.*:com.hp.hpl.jena.util.*:jena.cmdline:jena.util + com.hp.hpl.jena.shared.*:*.impl:com.hp.hpl.jena.assembler.assemblers:*.exceptions:*.regexptrees:com.hp.hpl.jena.mem:com.hp.hpl.jena.mem.*:com.hp.hpl.jena.n3:com.hp.hpl.jena.n3.*:com.hp.hpl.jena.rdf.arp.*:com.hp.hpl.jena.util.*:jena.cmdline:jena.util:org.apache.jena.impl.ext.* API - Application Programming Interface @@ -197,7 +213,47 @@ --> + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + + + + + com.google.guava:guava + + + + + com.google + org.apache.jena.ext.impl.com.google + + + + + diff --git a/jena-core/src/main/java/com/hp/hpl/jena/reasoner/rulesys/impl/LPBRuleEngine.java b/jena-core/src/main/java/com/hp/hpl/jena/reasoner/rulesys/impl/LPBRuleEngine.java index 8553f1dcccf..6533a352b7f 100644 --- a/jena-core/src/main/java/com/hp/hpl/jena/reasoner/rulesys/impl/LPBRuleEngine.java +++ b/jena-core/src/main/java/com/hp/hpl/jena/reasoner/rulesys/impl/LPBRuleEngine.java @@ -18,14 +18,33 @@ package com.hp.hpl.jena.reasoner.rulesys.impl; -import com.hp.hpl.jena.graph.*; -import com.hp.hpl.jena.reasoner.*; -import com.hp.hpl.jena.reasoner.rulesys.*; -import com.hp.hpl.jena.util.iterator.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +//In JAR these are shadowed as: +//import org.apache.jena.impl.ext.com.google.common.cache.Cache; +//import org.apache.jena.impl.ext.com.google.common.cache.CacheBuilder; + +import com.hp.hpl.jena.graph.Node; +import com.hp.hpl.jena.graph.Triple; +import com.hp.hpl.jena.reasoner.ReasonerException; +import com.hp.hpl.jena.reasoner.TriplePattern; +import com.hp.hpl.jena.reasoner.rulesys.BackwardRuleInfGraphI; +import com.hp.hpl.jena.reasoner.rulesys.Rule; +import com.hp.hpl.jena.util.iterator.ExtendedIterator; +import com.hp.hpl.jena.util.iterator.WrappedIterator; /** * LP version of the core backward chaining engine. For each parent inference @@ -52,11 +71,17 @@ public class LPBRuleEngine { protected boolean recordDerivations; /** List of engine instances which are still processing queries */ - protected List activeInterpreters = new ArrayList<>(); - + protected List activeInterpreters = new LinkedList<>(); + + protected final int MAX_CACHED_TABLED_GOALS = + Integer.getInteger("jena.rulesys.lp.max_cached_tabled_goals", 512*1024); + /** Table mapping tabled goals to generators for those goals. - * This is here so that partial goal state can be shared across multiple queries. */ - protected HashMap tabledGoals = new HashMap<>(); + * This is here so that partial goal state can be shared across multiple queries. + */ + protected Cache tabledGoals = CacheBuilder.newBuilder() + .maximumSize(MAX_CACHED_TABLED_GOALS).weakValues().build(); + //protected Map tabledGoals = new HashMap<>(); /** Set of generators waiting to be run */ protected LinkedList agenda = new LinkedList<>(); @@ -113,7 +138,7 @@ public synchronized ExtendedIterator find(TriplePattern goal) { */ public synchronized void reset() { checkSafeToUpdate(); - tabledGoals = new HashMap<>(); + tabledGoals.invalidateAll(); agenda.clear(); } @@ -252,19 +277,38 @@ public synchronized void tablePredicate(Node predicate) { /** * Return a generator for the given goal (assumes that the caller knows that * the goal should be tabled). + * + * Note: If an earlier Generator for the same goal exists in the + * cache, it will be returned without considering the provided clauses. + * * @param goal the goal whose results are to be generated * @param clauses the precomputed set of code blocks used to implement the goal */ - public synchronized Generator generatorFor(TriplePattern goal, List clauses) { - Generator generator = tabledGoals.get(goal); - if (generator == null) { - LPInterpreter interpreter = new LPInterpreter(this, goal, clauses, false); - activeInterpreters.add(interpreter); - generator = new Generator(interpreter, goal); - schedule(generator); - tabledGoals.put(goal, generator); - } - return generator; + public synchronized Generator generatorFor(final TriplePattern goal, final List clauses) { + try { + return tabledGoals.get(goal, new Callable() { + @Override + public Generator call() { + /** FIXME: Unify with #generatorFor(TriplePattern) - but investigate what about + * the edge case that this method might have been called with the of goal == null + * or goal.size()==0 -- which gives different behaviour in + * LPInterpreter constructor than through the route of + * generatorFor(TriplePattern) which calls a different LPInterpreter constructor + * which would fill in from RuleStore. + */ + LPInterpreter interpreter = new LPInterpreter(LPBRuleEngine.this, goal, clauses, false); + activeInterpreters.add(interpreter); + Generator generator = new Generator(interpreter, goal); + schedule(generator); + return generator; + } + }); + } catch (ExecutionException e) { + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException)e.getCause(); + } + throw new RuntimeException(e); + } } /** @@ -272,16 +316,28 @@ public synchronized Generator generatorFor(TriplePattern goal, List() { + @Override + public Generator call() { + LPInterpreter interpreter = new LPInterpreter(LPBRuleEngine.this, goal, false); + activeInterpreters.add(interpreter); + Generator generator = new Generator(interpreter, goal); + schedule(generator); + return generator; + } + }); + } catch (ExecutionException e) { + if (e.getCause() instanceof RuntimeException) { + throw (RuntimeException)e.getCause(); + } + throw new RuntimeException(e); + } + } + + long cachedTabledGoals() { + return tabledGoals.size(); } /** diff --git a/jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/impl/TestLPBRuleEngine.java b/jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/impl/TestLPBRuleEngine.java new file mode 100644 index 00000000000..15e35a3e97a --- /dev/null +++ b/jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/impl/TestLPBRuleEngine.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package com.hp.hpl.jena.reasoner.rulesys.impl; + +import java.lang.reflect.Field; +import java.util.List; + +import junit.framework.TestCase; +import junit.framework.TestSuite; + +import org.junit.Test; + +import com.hp.hpl.jena.graph.Factory; +import com.hp.hpl.jena.graph.Graph; +import com.hp.hpl.jena.graph.Node; +import com.hp.hpl.jena.graph.NodeFactory; +import com.hp.hpl.jena.graph.Triple; +import com.hp.hpl.jena.reasoner.rulesys.FBRuleInfGraph; +import com.hp.hpl.jena.reasoner.rulesys.FBRuleReasoner; +import com.hp.hpl.jena.reasoner.rulesys.Rule; +import com.hp.hpl.jena.util.iterator.ExtendedIterator; +import com.hp.hpl.jena.vocabulary.RDF; +import com.hp.hpl.jena.vocabulary.RDFS; + +public class TestLPBRuleEngine extends TestCase { + public static TestSuite suite() { + return new TestSuite(TestLPBRuleEngine.class, "TestLPBRuleEngine"); + } + + protected Node a = NodeFactory.createURI("a"); + protected Node p = NodeFactory.createURI("p"); + protected Node C1 = NodeFactory.createURI("C1"); + protected Node C2 = NodeFactory.createURI("C2"); + protected Node ty = RDF.Nodes.type; + + public FBRuleReasoner createReasoner(List rules) { + FBRuleReasoner reasoner = new FBRuleReasoner(rules); + reasoner.tablePredicate(RDFS.Nodes.subClassOf); + reasoner.tablePredicate(RDF.Nodes.type); + reasoner.tablePredicate(p); + return reasoner; + } + + @Test + public void testTabledGoalsCacheHits() throws Exception { + Graph data = Factory.createGraphMem(); + data.add(new Triple(a, ty, C1)); + List rules = Rule + .parseRules("[r1: (?x p ?t) <- (?x rdf:type C1), makeInstance(?x, p, C2, ?t)]" + + "[r2: (?t rdf:type C2) <- (?x rdf:type C1), makeInstance(?x, p, C2, ?t)]"); + + FBRuleInfGraph infgraph = (FBRuleInfGraph) createReasoner(rules).bind( + data); + + LPBRuleEngine engine = getEngineForGraph(infgraph); + assertEquals(0, engine.activeInterpreters.size()); + assertEquals(0, engine.tabledGoals.size()); + + ExtendedIterator it = infgraph.find(a, ty, C1); + while (it.hasNext()) { + it.next(); + // FIXME: Why do I need to consume all from the iterator + // to avoid leaking activeInterpreters? Calling .close() + // below should have been enough. + } + it.close(); + // how many were cached + assertEquals(1, engine.tabledGoals.size()); + // and no leaks of activeInterpreters + assertEquals(0, engine.activeInterpreters.size()); + + // Now ask again: + it = infgraph.find(a, ty, C1); + while (it.hasNext()) { + it.next(); + } + it.close(); + // if it was a cache hit, no change here: + assertEquals(1, engine.tabledGoals.size()); + assertEquals(0, engine.activeInterpreters.size()); + } + + @Test + public void testSaturateTabledGoals() throws Exception { + final int MAX = 1024; + // Set the cache size very small just for this test + System.setProperty("jena.rulesys.lp.max_cached_tabled_goals", "" + MAX); + try { + Graph data = Factory.createGraphMem(); + data.add(new Triple(a, ty, C1)); + List rules = Rule + .parseRules("[r1: (?x p ?t) <- (?x rdf:type C1), makeInstance(?x, p, C2, ?t)]" + + "[r2: (?t rdf:type C2) <- (?x rdf:type C1), makeInstance(?x, p, C2, ?t)]"); + + FBRuleInfGraph infgraph = (FBRuleInfGraph) createReasoner(rules) + .bind(data); + + LPBRuleEngine engine = getEngineForGraph(infgraph); + assertEquals(0, engine.activeInterpreters.size()); + assertEquals(0, engine.tabledGoals.size()); + + // JENA-901 + // Let's ask about lots of unknown subjects + for (int i = 0; i < MAX * 128; i++) { + Node test = NodeFactory.createURI("test" + i); + ExtendedIterator it = infgraph.find(test, ty, C2); + assertFalse(it.hasNext()); + it.close(); + } + + // Let's see how many were cached + assertEquals(MAX, engine.tabledGoals.size()); + // and no leaks of activeInterpreters (this will happen if we forget + // to call hasNext above) + assertEquals(0, engine.activeInterpreters.size()); + } finally { + System.clearProperty("jena.rulesys.lp.max_cached_tabled_goals"); + + } + } + + /** + * Use introspection to get to the LPBRuleEngine. + *

+ * We are crossing package boundaries and therefore this test would always + * be in the wrong package for either FBRuleInfGraph or LPBRuleEngine. + *

+ * This method should only be used for test purposes. + * + * @param infgraph + * @return + * @throws SecurityException + * @throws NoSuchFieldException + * @throws IllegalArgumentException + * @throws IllegalAccessException + */ + private LPBRuleEngine getEngineForGraph(FBRuleInfGraph infgraph) + throws NoSuchFieldException, SecurityException, + IllegalArgumentException, IllegalAccessException { + Field bEngine = FBRuleInfGraph.class.getDeclaredField("bEngine"); + bEngine.setAccessible(true); + LPBRuleEngine engine = (LPBRuleEngine) bEngine.get(infgraph); + return engine; + } + +} diff --git a/jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/test/TestPackage.java b/jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/test/TestPackage.java index 5334dc251d7..661a4906f7c 100755 --- a/jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/test/TestPackage.java +++ b/jena-core/src/test/java/com/hp/hpl/jena/reasoner/rulesys/test/TestPackage.java @@ -20,9 +20,12 @@ import junit.framework.TestSuite ; + import org.slf4j.Logger ; import org.slf4j.LoggerFactory ; +import com.hp.hpl.jena.reasoner.rulesys.impl.TestLPBRuleEngine; + /** * Aggregate tester that runs all the test associated with the rulesys package. */ @@ -45,6 +48,7 @@ private TestPackage() { addTest( "TestBackchainer", TestBackchainer.suite() ); addTest( "TestLPBasics", TestBasicLP.suite() ); addTest( "TestLPDerivation", TestLPDerivation.suite() ); + addTest( TestLPBRuleEngine.suite() ); addTest( "TestFBRules", TestFBRules.suite() ); addTest( "TestGenericRules", TestGenericRules.suite() ); addTest( "TestRETE", TestRETE.suite() ); diff --git a/jena-parent/pom.xml b/jena-parent/pom.xml index b573b626c25..fa608e8e9d3 100644 --- a/jena-parent/pom.xml +++ b/jena-parent/pom.xml @@ -665,6 +665,11 @@ 2.5.3 true + + org.apache.maven.plugins + maven-shade-plugin + 2.3 +