diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/BlankNodeFilteringGraph.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/BlankNodeFilteringGraph.java index f5a68d3483..521e039fb0 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/BlankNodeFilteringGraph.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/BlankNodeFilteringGraph.java @@ -24,6 +24,10 @@ public class BlankNodeFilteringGraph implements Graph { private Graph graph; + public Graph getInnerGraph() { + return graph; + } + public BlankNodeFilteringGraph(Graph graph) { this.graph = graph; } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceModelMaker.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceModelMaker.java index 8447477dee..d2e0db56ee 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceModelMaker.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/dao/jena/RDFServiceModelMaker.java @@ -103,7 +103,7 @@ public Model openModel(String name, boolean strict) { @Override public void removeModel(String name) { Model m = getModel(name); - m.removeAll(null, null, null); + m.removeAll(); } @Override diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkGraphMem.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkGraphMem.java new file mode 100644 index 0000000000..ef13daa736 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkGraphMem.java @@ -0,0 +1,17 @@ +package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters; + +import org.apache.jena.graph.Triple; +import org.apache.jena.mem.GraphMem; + +public class BulkGraphMem extends GraphMem { + + public void addWithoutNotify(Triple t) { + checkOpen(); + performAdd(t); + } + + public final void deleteWithoutNotify(Triple t) { + checkOpen(); + performDelete(t); + } +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkModelCom.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkModelCom.java new file mode 100644 index 0000000000..5a548a26ee --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkModelCom.java @@ -0,0 +1,34 @@ +package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters; + +import org.apache.jena.graph.Graph; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.impl.ModelCom; + +public class BulkModelCom extends ModelCom { + + public BulkModelCom(Graph graph) { + super(graph); + } + + @Override + public Model remove(Model m) { + Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph); + if (unwrappedGraph instanceof BulkGraphMem) { + GraphUtils.deleteFrom((BulkGraphMem) unwrappedGraph, m.getGraph()); + } else { + super.remove(m); + } + return this; + } + + @Override + public Model add(Model m) { + Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph); + if (unwrappedGraph instanceof BulkGraphMem) { + GraphUtils.addInto((BulkGraphMem) unwrappedGraph, m.getGraph()); + } else { + super.add(m); + } + return this; + } +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkOntModelImpl.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkOntModelImpl.java new file mode 100644 index 0000000000..3d925a4b02 --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkOntModelImpl.java @@ -0,0 +1,39 @@ +package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters; + +import org.apache.jena.graph.Graph; +import org.apache.jena.ontology.OntModelSpec; +import org.apache.jena.ontology.impl.OntModelImpl; +import org.apache.jena.rdf.model.Model; + +public class BulkOntModelImpl extends OntModelImpl { + + public BulkOntModelImpl(OntModelSpec spec) { + super(spec); + } + + public BulkOntModelImpl(OntModelSpec owlMem, Model bareModel) { + super(owlMem, bareModel); + } + + @Override + public Model remove(Model m) { + Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph); + if (unwrappedGraph instanceof BulkGraphMem) { + GraphUtils.deleteFrom((BulkGraphMem) unwrappedGraph, m.getGraph()); + } else { + super.remove(m); + } + return this; + } + + @Override + public Model add(Model m) { + Graph unwrappedGraph = GraphUtils.unwrapUnionGraphs(graph); + if (unwrappedGraph instanceof BulkGraphMem) { + GraphUtils.addInto((BulkGraphMem) unwrappedGraph, m.getGraph()); + } else { + super.add(m); + } + return this; + } +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingModel.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingModel.java index 6952959336..2657c6c660 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingModel.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingModel.java @@ -4,6 +4,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph; import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph; +import edu.cornell.mannlib.vitro.webapp.rdfservice.adapters.VitroModelFactory.BulkUpdatingUnion; + import org.apache.jena.graph.Graph; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; @@ -22,30 +24,20 @@ public class BulkUpdatingModel extends AbstractModelDecorator { private static final RDFReaderF readerFactory = new RDFReaderFImpl(); - private AbstractBulkUpdater updater; + protected AbstractBulkUpdater updater; - protected BulkUpdatingModel(Model m) { + public BulkUpdatingModel(Model m) { super(m); - if (m instanceof BulkUpdatingModel) { + Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph()); + if(graph instanceof BulkUpdatingUnion){ + updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph); + return; + } + if (m instanceof BulkUpdatingOntModel) { + this.updater = ((BulkUpdatingOntModel) m).updater; + } else if (m instanceof BulkUpdatingModel) { this.updater = ((BulkUpdatingModel) m).updater; } else { - Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph()); - if (graph instanceof RDFServiceGraph) { - updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph); - } else if (graph instanceof SparqlGraph) { - updater = new SparqlBulkUpdater((SparqlGraph) graph); - } else { - updater = null; - } - } - } - - protected BulkUpdatingModel(Model m, Model baseModel) { - super(m); - if (baseModel instanceof BulkUpdatingModel) { - this.updater = ((BulkUpdatingModel) baseModel).updater; - } else { - Graph graph = GraphUtils.unwrapUnionGraphs(baseModel.getGraph()); if (graph instanceof RDFServiceGraph) { updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph); } else if (graph instanceof SparqlGraph) { diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingOntModel.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingOntModel.java index 8eb1521fee..44a5582b87 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingOntModel.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/BulkUpdatingOntModel.java @@ -4,6 +4,8 @@ import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph; import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph; +import edu.cornell.mannlib.vitro.webapp.rdfservice.adapters.VitroModelFactory.BulkUpdatingUnion; + import org.apache.jena.graph.Graph; import org.apache.jena.ontology.OntModel; import org.apache.jena.rdf.model.Model; @@ -23,30 +25,20 @@ public class BulkUpdatingOntModel extends AbstractOntModelDecorator { private static final RDFReaderF readerFactory = new RDFReaderFImpl(); - private AbstractBulkUpdater updater; + protected AbstractBulkUpdater updater; protected BulkUpdatingOntModel(OntModel m) { super(m); + Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph()); + if(graph instanceof BulkUpdatingUnion){ + updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph); + return; + } if (m instanceof BulkUpdatingOntModel) { this.updater = ((BulkUpdatingOntModel) m).updater; + } else if (m instanceof BulkUpdatingModel) { + this.updater = ((BulkUpdatingModel) m).updater; } else { - Graph graph = GraphUtils.unwrapUnionGraphs(m.getGraph()); - if (graph instanceof RDFServiceGraph) { - updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph); - } else if (graph instanceof SparqlGraph) { - updater = new SparqlBulkUpdater((SparqlGraph) graph); - } else { - updater = null; - } - } - } - - protected BulkUpdatingOntModel(OntModel m, OntModel baseModel) { - super(m); - if (baseModel instanceof BulkUpdatingOntModel) { - this.updater = ((BulkUpdatingOntModel) baseModel).updater; - } else { - Graph graph = GraphUtils.unwrapUnionGraphs(baseModel.getGraph()); if (graph instanceof RDFServiceGraph) { updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph); } else if (graph instanceof SparqlGraph) { diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/GraphUtils.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/GraphUtils.java index d681ad27b1..978641211a 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/GraphUtils.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/GraphUtils.java @@ -2,20 +2,128 @@ package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters; +import org.apache.jena.atlas.iterator.Iter; import org.apache.jena.graph.Graph; +import org.apache.jena.graph.Triple; import org.apache.jena.graph.compose.MultiUnion; +import org.apache.jena.util.IteratorCollection; +import org.apache.jena.util.iterator.ExtendedIterator; +import java.util.ArrayList; +import java.util.Iterator; import java.util.List; final public class GraphUtils { - public static Graph unwrapUnionGraphs(Graph graph) { - if (graph != null && graph instanceof MultiUnion) { - List subGraphs = ((MultiUnion)graph).getSubGraphs(); - if (subGraphs == null || subGraphs.isEmpty()) { - return ((MultiUnion)graph).getBaseGraph(); - } - } - - return graph; - } + + private static final int CMP_GREATER = 1; + private static final int CMP_EQUAL = 0; + private static final int CMP_LESS = -1; + private static int MIN_SRC_SIZE = 1000; + // If source and destination are large, limit the search for the best way round + // to "deleteFrom" + private static int DST_SRC_RATIO = 2; + + public static Graph unwrapUnionGraphs(Graph graph) { + if (graph != null && graph instanceof MultiUnion) { + List subGraphs = ((MultiUnion) graph).getSubGraphs(); + if (subGraphs == null || subGraphs.isEmpty()) { + return ((MultiUnion) graph).getBaseGraph(); + } + } + return graph; + } + + public static void deleteFrom(BulkGraphMem bulkGraphMem, Graph srcGraph) { + boolean events = bulkGraphMem.getEventManager().listening(); + if (bulkGraphMem == srcGraph && !events) { + bulkGraphMem.clear(); + return; + } + boolean loopOnSrc = decideHowtoExecuteBySizeStep(bulkGraphMem, srcGraph); + if (loopOnSrc) { + deleteLoopSrc(bulkGraphMem, srcGraph); + return; + } + deleteLoopDst(bulkGraphMem, srcGraph); + } + + public static void addInto(BulkGraphMem bulkGraphMem, Graph srcGraph) { + if (bulkGraphMem == srcGraph && !bulkGraphMem.getEventManager().listening()) { + return; + } + bulkGraphMem.getPrefixMapping().setNsPrefixes(srcGraph.getPrefixMapping()); + addIteratorWorker(bulkGraphMem, findAll(srcGraph)); + bulkGraphMem.getEventManager().notifyAddGraph(bulkGraphMem, srcGraph); + } + + private static ExtendedIterator findAll(Graph g) { + return g.find(); + } + + private static void addIteratorWorker(BulkGraphMem bulkGraphMem, Iterator it) { + List s = IteratorCollection.iteratorToList(it); + addIteratorWorkerDirect(bulkGraphMem, s.iterator()); + } + + private static void addIteratorWorkerDirect(BulkGraphMem bulkGraphMem, Iterator it) { + it.forEachRemaining(bulkGraphMem::addWithoutNotify); + } + + private static void deleteLoopSrc(BulkGraphMem bulkGraphMem, Graph srcGraph) { + deleteIteratorWorker(bulkGraphMem, findAll(srcGraph)); + bulkGraphMem.getEventManager().notifyDeleteGraph(bulkGraphMem, srcGraph); + } + + private static void deleteLoopDst(BulkGraphMem bulkGraphMem, Graph srcGraph) { + // Size the list to avoid reallocation on growth. + int dstSize = bulkGraphMem.size(); + List toBeDeleted = new ArrayList<>(dstSize); + + Iterator iter = findAll(bulkGraphMem); + for (; iter.hasNext();) { + Triple t = iter.next(); + if (srcGraph.contains(t)) { + toBeDeleted.add(t); + } + } + deleteIteratorWorkerDirect(bulkGraphMem, toBeDeleted.iterator()); + bulkGraphMem.getEventManager().notifyDeleteGraph(bulkGraphMem, srcGraph); + } + + private static void deleteIteratorWorker(BulkGraphMem bulkGraphMem, Iterator it) { + List s = IteratorCollection.iteratorToList(it); + deleteIteratorWorkerDirect(bulkGraphMem, s.iterator()); + } + + private static void deleteIteratorWorkerDirect(BulkGraphMem bulkGraphMem, Iterator it) { + it.forEachRemaining(bulkGraphMem::deleteWithoutNotify); + } + + private static boolean decideHowtoExecuteBySizeStep(BulkGraphMem bulkGraphMem, Graph srcGraph) { + int srcSize = srcGraph.size(); + if (srcSize <= MIN_SRC_SIZE) + return true; + boolean loopOnSrc = (srcSize <= MIN_SRC_SIZE + || compareSizeTo(bulkGraphMem, DST_SRC_RATIO * srcSize) == CMP_GREATER); + return loopOnSrc; + } + + private static int compareSizeTo(Graph graph, int size) { + ExtendedIterator it = graph.find(); + try { + int stepsTake = Iter.step(it, size); + if (stepsTake < size) { + // Iterator ran out. + return CMP_LESS; + } + if (!it.hasNext()) { + // Finished at the same time. + return CMP_EQUAL; + } + // Still more to go + return CMP_GREATER; + } finally { + it.close(); + } + } } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUnionUpdater.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUnionUpdater.java new file mode 100644 index 0000000000..d927c6575c --- /dev/null +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUnionUpdater.java @@ -0,0 +1,118 @@ +/* $This file is distributed under the terms of the license in LICENSE$ */ + +package edu.cornell.mannlib.vitro.webapp.rdfservice.adapters; + +import edu.cornell.mannlib.vitro.webapp.dao.jena.BlankNodeFilteringGraph; +import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph; +import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph; +import edu.cornell.mannlib.vitro.webapp.rdfservice.adapters.VitroModelFactory.BulkUpdatingUnion; + +import org.apache.jena.graph.Graph; +import org.apache.jena.rdf.model.Model; + +public class RDFServiceBulkUnionUpdater extends AbstractBulkUpdater { + + private AbstractBulkUpdater leftUpdater; + private AbstractBulkUpdater rightUpdater; + private Model baseModel; + private Model plusModel; + + public RDFServiceBulkUnionUpdater(Model baseModel, Model plusModel) { + this.baseModel = baseModel; + this.plusModel = plusModel; + if (baseModel != null) { + leftUpdater = getUpdater(baseModel); + } + if (plusModel != null) { + rightUpdater = getUpdater(plusModel); + } + } + + public RDFServiceBulkUnionUpdater(BulkUpdatingUnion union) { + baseModel = union.getBaseModel(); + if (baseModel != null) { + leftUpdater = getUpdater(baseModel); + } + plusModel = union.getPlusModel(); + if (plusModel != null) { + rightUpdater = getUpdater(plusModel); + } + } + + private AbstractBulkUpdater getUpdater(Model model) { + AbstractBulkUpdater updater = null; + if (model instanceof BulkUpdatingOntModel) { + updater = ((BulkUpdatingOntModel) model).updater; + return updater; + } else if (model instanceof BulkUpdatingModel) { + updater = ((BulkUpdatingModel) model).updater; + return updater; + } + Graph graph = GraphUtils.unwrapUnionGraphs(model.getGraph()); + if (graph instanceof BulkUpdatingUnion) { + updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph); + } else if (graph instanceof RDFServiceGraph) { + updater = new RDFServiceBulkUpdater((RDFServiceGraph) graph); + } else if (hasBlankNodeFilterginGraphInnerRdfServiceGraph(graph)) { + BlankNodeFilteringGraph blankNodeGraph = (BlankNodeFilteringGraph) graph; + RDFServiceGraph rdfServiceGraph = (RDFServiceGraph)blankNodeGraph.getInnerGraph(); + updater = new RDFServiceBulkUpdater(rdfServiceGraph); + } else if (graph instanceof BulkUpdatingUnion) { + updater = new RDFServiceBulkUnionUpdater((BulkUpdatingUnion) graph); + } else if (graph instanceof SparqlGraph) { + updater = new SparqlBulkUpdater((SparqlGraph) graph); + } else { + updater = null; + } + return updater; + } + + private boolean hasBlankNodeFilterginGraphInnerRdfServiceGraph(Graph graph) { + if (!(graph instanceof BlankNodeFilteringGraph)) { + return false; + } + BlankNodeFilteringGraph blankNodeGraph = (BlankNodeFilteringGraph) graph; + final Graph innerGraph = blankNodeGraph.getInnerGraph(); + if (innerGraph instanceof RDFServiceGraph) { + return true; + } + return false; + } + + @Override + protected void performAddModel(Model model) { + if (leftUpdater != null) { + leftUpdater.performAddModel(model); + } else if (baseModel != null) { + baseModel.add(model); + } + } + + @Override + protected void performRemoveModel(Model model) { + if (leftUpdater != null) { + leftUpdater.performRemoveModel(model); + } else if (baseModel != null) { + baseModel.remove(model); + } + if (rightUpdater != null) { + rightUpdater.performRemoveModel(model); + } else if (plusModel != null) { + plusModel.remove(model); + } + } + + @Override + protected void performRemoveAll() { + if (leftUpdater != null) { + leftUpdater.performRemoveAll(); + } else if (baseModel != null) { + baseModel.removeAll(); + } + if (rightUpdater != null) { + rightUpdater.performRemoveAll(); + } else if (plusModel != null) { + plusModel.removeAll(); + } + } +} diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUpdater.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUpdater.java index 6a8a727038..23a80cbd1d 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUpdater.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/RDFServiceBulkUpdater.java @@ -6,15 +6,20 @@ import edu.cornell.mannlib.vitro.webapp.rdfservice.ChangeSet; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFService; import edu.cornell.mannlib.vitro.webapp.rdfservice.RDFServiceException; -import edu.cornell.mannlib.vitro.webapp.rdfservice.impl.RDFServiceUtils; import org.apache.jena.graph.GraphEvents; import org.apache.jena.rdf.model.Model; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.InputStream; public class RDFServiceBulkUpdater extends AbstractBulkUpdater { RDFServiceGraph graph; + private static final int chunkSize = 50000; + private static final String REMOVE_CHUNK_QUERY = + "CONSTRUCT { ?s ?p ?o } " + + "WHERE { ?s ?p ?o } " + + "LIMIT " + chunkSize; public RDFServiceBulkUpdater(RDFServiceGraph graph) { this.graph = graph; @@ -53,49 +58,35 @@ protected void performRemoveModel(Model model) { @Override protected void performRemoveAll() { String graphURI = graph.getGraphURI(); - - String findPattern = "?s ?p ?o"; - - StringBuilder findQuery = new StringBuilder("CONSTRUCT { ") - .append(findPattern) - .append(" } WHERE { \n"); + String queryString; if (graphURI != null) { - findQuery.append(" GRAPH <").append(graphURI).append("> { "); + queryString = getRemoveGraphChunkPattern(graphURI); + } else { + queryString = REMOVE_CHUNK_QUERY; } - findQuery.append(findPattern); - if (graphURI != null) { - findQuery.append(" } "); - } - findQuery.append("\n}"); - - String queryString = findQuery.toString(); - - int chunkSize = 50000; - boolean done = false; - - while (!done) { - String chunkQueryString = queryString + " LIMIT " + chunkSize; - + while (!graph.isEmpty()) { try { - Model chunkToRemove = RDFServiceUtils.parseModel( - graph.getRDFService().sparqlConstructQuery( - chunkQueryString, RDFService.ModelSerializationFormat.N3), - RDFService.ModelSerializationFormat.N3); - if (chunkToRemove.size() > 0) { - ChangeSet cs = graph.getRDFService().manufactureChangeSet(); - ByteArrayOutputStream out = new ByteArrayOutputStream(); - chunkToRemove.write(out, "N-TRIPLE"); - cs.addRemoval(new ByteArrayInputStream(out.toByteArray()), - RDFService.ModelSerializationFormat.N3, graphURI); - graph.getRDFService().changeSetUpdate(cs); - } else { - done = true; - } + InputStream chunkToRemove = graph.getRDFService().sparqlConstructQuery( + queryString, RDFService.ModelSerializationFormat.N3); + ChangeSet cs = graph.getRDFService().manufactureChangeSet(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + cs.addRemoval(chunkToRemove, RDFService.ModelSerializationFormat.N3, graphURI); + graph.getRDFService().changeSetUpdate(cs); } catch (RDFServiceException e) { throw new RuntimeException(e); } } - graph.getEventManager().notifyEvent(graph, GraphEvents.removeAll); } + + private static String getRemoveGraphChunkPattern(String uri) { + return + "CONSTRUCT { ?s ?p ?o } " + + "WHERE { " + + "GRAPH <" + uri + "> {" + + "?s ?p ?o " + + "} "+ + "} "+ + "LIMIT " + chunkSize; + } } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactory.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactory.java index 4b5dbd2ad8..e129ef7587 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactory.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactory.java @@ -4,24 +4,16 @@ import static org.apache.jena.ontology.OntModelSpec.OWL_MEM; -import edu.cornell.mannlib.vitro.webapp.dao.jena.RDFServiceGraph; -import edu.cornell.mannlib.vitro.webapp.dao.jena.SparqlGraph; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; - import org.apache.jena.graph.Graph; -import org.apache.jena.graph.compose.MultiUnion; import org.apache.jena.graph.compose.Union; import org.apache.jena.ontology.OntModel; -import org.apache.jena.ontology.impl.OntModelImpl; import org.apache.jena.rdf.model.Model; import org.apache.jena.rdf.model.ModelFactory; -import org.apache.jena.rdf.model.impl.ModelCom; import edu.cornell.mannlib.vitro.webapp.utils.logging.ToString; -import java.util.List; - /** * Make models that will do proper bulk updates. */ @@ -29,52 +21,49 @@ public class VitroModelFactory { private static final Log log = LogFactory.getLog(VitroModelFactory.class); public static Model createModel() { - return ModelFactory.createDefaultModel(); + return createBulkInMemoryModel(); + } + + private static BulkModelCom createBulkInMemoryModel() { + return new BulkModelCom(new BulkGraphMem()); } public static OntModel createOntologyModel() { - return ModelFactory.createOntologyModel(OWL_MEM); + return new BulkOntModelImpl(OWL_MEM, createBulkInMemoryModel()); } public static OntModel createOntologyModel(Model model) { Graph graph = model.getGraph(); - Model bareModel = new ModelCom(graph); - OntModel ontModel = new OntModelImpl(OWL_MEM, bareModel); + Model bareModel = new BulkModelCom(graph); + OntModel ontModel = new BulkOntModelImpl(OWL_MEM, bareModel); return new BulkUpdatingOntModel(ontModel); } public static Model createUnion(Model baseModel, Model plusModel) { - Graph baseGraph = baseModel.getGraph(); - Graph plusGraph = plusModel.getGraph(); - - BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseGraph, plusGraph); - + BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseModel, plusModel); Model unionModel = ModelFactory.createModelForGraph(unionGraph); - - return new BulkUpdatingModel(unionModel, baseModel); + return new BulkUpdatingModel(unionModel); } public static OntModel createUnion(OntModel baseModel, OntModel plusModel) { - Graph baseGraph = baseModel.getGraph(); - Graph plusGraph = plusModel.getGraph(); - - BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseGraph, plusGraph); - + BulkUpdatingUnion unionGraph = new BulkUpdatingUnion(baseModel, plusModel); Model unionModel = ModelFactory.createModelForGraph(unionGraph); OntModel unionOntModel = ModelFactory.createOntologyModel(OWL_MEM, unionModel); - - - return new BulkUpdatingOntModel(unionOntModel, baseModel); + return new BulkUpdatingOntModel(unionOntModel); } public static Model createModelForGraph(Graph g) { return new BulkUpdatingModel(ModelFactory.createModelForGraph(g)); } - private static class BulkUpdatingUnion extends Union { - @SuppressWarnings("deprecation") - public BulkUpdatingUnion(Graph L, Graph R) { - super(L, R); + public static class BulkUpdatingUnion extends Union { + private Model baseModel; + private Model plusModel; + + public BulkUpdatingUnion(Model baseModel, Model plusModel) { + super(baseModel.getGraph(), plusModel.getGraph()); + this.baseModel = baseModel; + this.plusModel = plusModel; } @Override @@ -83,5 +72,13 @@ public String toString() { + ToString.graphToString(L) + ", R=" + ToString.graphToString(R) + "]"; } + + public Model getBaseModel() { + return baseModel; + } + + public Model getPlusModel() { + return plusModel; + } } } diff --git a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java index c8535c76f3..bcf25174d7 100644 --- a/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java +++ b/api/src/main/java/edu/cornell/mannlib/vitro/webapp/servlet/setup/RDFFilesLoader.java @@ -18,6 +18,7 @@ import java.util.TreeSet; import edu.cornell.mannlib.vitro.webapp.i18n.selection.SelectedLocale; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -182,12 +183,16 @@ private static Set getPaths(String parentDir, String... strings) { private static void readOntologyFileIntoModel(Path p, Model model) { String format = getRdfFormat(p); log.debug("Loading " + p); + Model memModel = ModelFactory.createDefaultModel(); try (InputStream stream = new FileInputStream(p.toFile())) { - model.read(stream, null, format); + memModel.read(stream, null, format); + model.add(memModel); log.debug("...successful"); } catch (Exception e) { log.warn("Could not load file '" + p + "' as " + format + ". Check that it contains valid data.", e); + } finally { + memModel.close(); } } diff --git a/api/src/test/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactoryTest.java b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactoryTest.java index 2054761595..4724bb3610 100644 --- a/api/src/test/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactoryTest.java +++ b/api/src/test/java/edu/cornell/mannlib/vitro/webapp/rdfservice/adapters/VitroModelFactoryTest.java @@ -346,10 +346,13 @@ public void addMultipleToUnion() { public void addMultipleToVitroUnion() { umg = new VitroUnionModelGroup(); umg.m.add(MULTIPLE_STATEMENTS); - new MethodCalls().add(umg.base.g, "add", "add") - .add(umg.base.l, "addedStatement", "addedStatement") - .add(umg.plus.g).add(umg.plus.l) - .add(umg.l, "addedStatements").test(); + new MethodCalls() + .add(umg.base.g, "performAdd", "performAdd") + .add(umg.base.l, "addedStatements", "addedStatements") + .add(umg.plus.g) + .add(umg.plus.l) + .add(umg.l) + .test(); } // ---------------------------------------------------------------------- @@ -452,10 +455,12 @@ public void addMultipleToOntUnion() { public void addMultipleToVitroOntUnion() { uomg = new VitroUnionOntModelGroup(); uomg.om.add(MULTIPLE_STATEMENTS); - new MethodCalls().add(uomg.base.g, "add", "add") + new MethodCalls() + .add(uomg.base.g, "add", "add") .add(uomg.base.l, "addedStatement", "addedStatement") .add(uomg.plus.g).add(uomg.plus.l) - .add(uomg.l, "addedStatements").test(); + .add(uomg.l) + .test(); } // ---------------------------------------------------------------------- @@ -566,12 +571,16 @@ public void addMultipleToOntModeledUnionModel() { public void addMultipleToVitroOntModeledUnionModel() { omumg = new VitroOntModelUnionModelGroup(); omumg.om.add(MULTIPLE_STATEMENTS); - new MethodCalls().add(omumg.om, "add").add(omumg.ol, "addedStatements") - .add(omumg.union.base.g, "add", "add") - .add(omumg.union.base.m) - .add(omumg.union.base.l, "addedStatement", "addedStatement") + new MethodCalls() + .add(omumg.om, "add") + .add(omumg.ol) + .add(omumg.union.base.g, "performAdd", "performAdd") + .add(omumg.union.base.m, "add", "add") + .add(omumg.union.base.l, "addedStatements", "addedStatements") .add(omumg.union.plus.g) - .add(omumg.union.plus.m).add(omumg.union.plus.l).test(); + .add(omumg.union.plus.m) + .add(omumg.union.plus.l) + .test(); } // ----------------------------------------------------------------------