diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b6cade56..902574e0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ ### Added +- Store adds interface support for closing the Store cleanly and releasing underlying connections. +- SparqlStoreImpl can now be set up with a custom query executor + ### Changed - Update SHACLEX from 0.0.87 to 0.1.93 (breaking change but should not affect the consumers of Lyo Validation) diff --git a/store/store-core/src/main/java/org/eclipse/lyo/store/Store.java b/store/store-core/src/main/java/org/eclipse/lyo/store/Store.java index 04e79c71f..75c6a3494 100644 --- a/store/store-core/src/main/java/org/eclipse/lyo/store/Store.java +++ b/store/store-core/src/main/java/org/eclipse/lyo/store/Store.java @@ -166,12 +166,12 @@ List getResources(URI namedGraphUri, Class clazz, in * @throws ModelUnmarshallingException if the classes cannot be instantiated or another error * occurred when working with Jena model. */ - List getResources(URI namedGraphUri, Class clazz, - String prefixes, String where, String searchTerms, + List getResources(URI namedGraphUri, Class clazz, + String prefixes, String where, String searchTerms, int limit, int offset) throws StoreAccessException, ModelUnmarshallingException; /** - * Retrieve a Jena model that satisfies the given where parameter as defined in the OSLC Query language (https://tools.oasis-open.org/version-control/svn/oslc-core/trunk/specs/oslc-query.html) + * Retrieve a Jena model that satisfies the given where parameter as defined in the OSLC Query language (https://tools.oasis-open.org/version-control/svn/oslc-core/trunk/specs/oslc-query.html) * If the namedGraph is null, the query is applied on all namedGraph in the triplestore. * The method currently only provides support for terms of type Comparisons, where the operator is 'EQUALS', and the operand is either a String or a URI. * @@ -187,7 +187,7 @@ List getResources(URI namedGraphUri, Class clazz, Model getResources(URI namedGraph, String prefixes, String where, int limit, int offset); /** - * Retrieve a Jena model that satisfies the given where parameter as defined in the OSLC Query language (https://tools.oasis-open.org/version-control/svn/oslc-core/trunk/specs/oslc-query.html) + * Retrieve a Jena model that satisfies the given where parameter as defined in the OSLC Query language (https://tools.oasis-open.org/version-control/svn/oslc-core/trunk/specs/oslc-query.html) * If the namedGraph is null, the query is applied on all namedGraph in the triplestore. * The method currently only provides support for terms of type Comparisons, where the operator is 'EQUALS', and the operand is either a String or a URI. * @@ -312,4 +312,11 @@ default boolean appendResource(URI namedGraphUri, final T * @since 0.23.0 */ void removeAll(); + + /** + * Close connection + * + * @since 4.1.0 + */ + void close(); } diff --git a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/JenaTdbStoreImpl.java b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/JenaTdbStoreImpl.java index 28d084c3f..acc68edeb 100644 --- a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/JenaTdbStoreImpl.java +++ b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/JenaTdbStoreImpl.java @@ -207,6 +207,12 @@ public void removeAll() { } } + @Override + public void close() { + TDB.sync(dataset); + dataset.close(); + } + @Override public T getResource(final URI namedGraph, final URI uri, final Class clazz) diff --git a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/SparqlStoreImpl.java b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/SparqlStoreImpl.java index 1899dd10c..73a3ec65d 100644 --- a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/SparqlStoreImpl.java +++ b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/SparqlStoreImpl.java @@ -274,7 +274,7 @@ public List getResources(final URI namedGraph, final Cl public Model getResources(final URI namedGraph, final String prefixes, final String where, final int limit, final int offset) { return getResources(namedGraph, prefixes, where, null, limit, offset); } - + @Override public Model getResources(final URI namedGraph, final String prefixes, final String where, final String searchTerms, final int limit, final int offset) { @@ -305,7 +305,7 @@ public Model getResources(final URI namedGraph, final String prefixes, final Str Query describeQuery = describeBuilder.build() ; String describeQueryString = describeQuery.toString(); final QueryExecution queryExecution = queryExecutor.prepareSparqlQuery(describeQueryString); - + Model execDescribe; try { execDescribe = queryExecution.execDescribe(); @@ -377,6 +377,12 @@ public void removeAll() { queryExecutor.prepareSparqlUpdate("CLEAR ALL").execute(); } + @Override + public void close() { + queryExecutor.release(); + log.debug("Underlying SPARQL connection has been released"); + } + private String oslcQueryPrefixes(final Class clazz) { return "rdf=" + "<" + org.apache.jena.vocabulary.RDF.uri + ">"; } @@ -454,7 +460,7 @@ private Model modelFromQueryByUri(final URI namedGraph, final URI uri) { throw e; } return execDescribe; - + } private Model modelFromQueryFlatPaged(final URI namedGraph, final URI type, final int limit, @@ -528,7 +534,7 @@ private SelectBuilder constructSparqlWhere(final String prefixes, final String w SimpleTerm simpleTerm = iterator.next(); Type termType = simpleTerm.type(); PName property = simpleTerm.property(); - + if (!termType.equals(Type.COMPARISON)){ throw new UnsupportedOperationException("only support for terms of type Comparisons"); } @@ -536,7 +542,7 @@ private SelectBuilder constructSparqlWhere(final String prefixes, final String w if (!aComparisonTerm.operator().equals(Operator.EQUALS)){ throw new UnsupportedOperationException("only support for terms of type Comparisons, where the operator is 'EQUALS'"); } - + Value comparisonOperand = aComparisonTerm.operand(); Value.Type operandType = comparisonOperand.type(); String predicate; @@ -546,7 +552,7 @@ private SelectBuilder constructSparqlWhere(final String prefixes, final String w else { predicate = property.toString(); } - + switch (operandType) { case DECIMAL: DecimalValue decimalOperand = (DecimalValue) comparisonOperand; @@ -568,7 +574,7 @@ private SelectBuilder constructSparqlWhere(final String prefixes, final String w } catch (ParseException e) { throw new IllegalArgumentException("whereExpression could not be parsed", e); } - + //Setup searchTerms //Add a sparql filter "FILTER regex(?o, "", "i")" to the distinctResourcesQuery if (!StringUtils.isEmpty(searchTerms)) { @@ -576,14 +582,14 @@ private SelectBuilder constructSparqlWhere(final String prefixes, final String w E_Regex regex = factory.regex(factory.str("?o"), searchTerms, "i"); distinctResourcesQuery.addFilter(regex); } - + if (limit > 0) { distinctResourcesQuery.setLimit(limit); } if (offset > 0) { distinctResourcesQuery.setOffset(offset); } - + SelectBuilder constructSelectQuery = new SelectBuilder(); constructSelectQuery.addVar( "s p o" ) .addSubQuery(distinctResourcesQuery); diff --git a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/DatasetQueryExecutorImpl.java b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/DatasetQueryExecutorImpl.java index 506a5bdc2..078bbd4c1 100644 --- a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/DatasetQueryExecutorImpl.java +++ b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/DatasetQueryExecutorImpl.java @@ -17,6 +17,7 @@ import org.apache.jena.query.Dataset; import org.apache.jena.query.QueryExecution; import org.apache.jena.query.QueryExecutionFactory; +import org.apache.jena.tdb.TDB; import org.apache.jena.tdb.TDBFactory; import org.apache.jena.update.GraphStore; import org.apache.jena.update.GraphStoreFactory; @@ -40,7 +41,7 @@ public class DatasetQueryExecutorImpl implements JenaQueryExecutor { private static final Logger log = LoggerFactory.getLogger(DatasetQueryExecutorImpl.class); private final Dataset dataset; - private final GraphStore graphStore; + private volatile boolean released = false; /** * Use {@link StoreFactory} instead. @@ -49,21 +50,33 @@ public DatasetQueryExecutorImpl() { this(TDBFactory.createDataset()); } - DatasetQueryExecutorImpl(final Dataset dataset) { + public DatasetQueryExecutorImpl(final Dataset dataset) { this.dataset = dataset; - this.graphStore = GraphStoreFactory.create(dataset); } @Override public QueryExecution prepareSparqlQuery(final String query) { + if(released) { + throw new IllegalStateException("Cannot execute queries after releasing the connection"); + } log.debug("Running query: '{}'", query); return QueryExecutionFactory.create(query, dataset); } @Override public UpdateProcessor prepareSparqlUpdate(final String query) { + if(released) { + throw new IllegalStateException("Cannot execute queries after releasing the connection"); + } log.debug("Running update: '{}'", query); final UpdateRequest update = UpdateFactory.create(query); - return UpdateExecutionFactory.create(update, graphStore); + return UpdateExecutionFactory.create(update, dataset); + } + + @Override + public void release() { + TDB.sync(dataset); + released = true; + dataset.close(); } } diff --git a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/JenaQueryExecutor.java b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/JenaQueryExecutor.java index 4fbbc982b..7a8e9a651 100644 --- a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/JenaQueryExecutor.java +++ b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/JenaQueryExecutor.java @@ -41,4 +41,9 @@ public interface JenaQueryExecutor { * @return prepared processor */ UpdateProcessor prepareSparqlUpdate(String query); + + /** + * Release a connection to the underlying engine + */ + void release(); } diff --git a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorBasicAuthImpl.java b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorBasicAuthImpl.java index 9c4518973..6df8cfa89 100644 --- a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorBasicAuthImpl.java +++ b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorBasicAuthImpl.java @@ -25,6 +25,10 @@ import org.apache.jena.update.UpdateExecutionFactory; import org.apache.jena.update.UpdateFactory; import org.apache.jena.update.UpdateProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; /** * SparqlQueryExecutorImpl is a SPARQL endpoint-based implementation of {@link JenaQueryExecutor}. @@ -34,10 +38,12 @@ * @since 0.14.0 */ public class SparqlQueryExecutorBasicAuthImpl implements JenaQueryExecutor { + private final Logger log = LoggerFactory.getLogger(SparqlQueryExecutorBasicAuthImpl.class); private final String queryEndpoint; private final String updateEndpoint; private final CloseableHttpClient client; + private volatile boolean released = false; public SparqlQueryExecutorBasicAuthImpl(final String sparqlEndpoint, final String updateEndpoint, final String login, final String password) { @@ -52,15 +58,32 @@ public SparqlQueryExecutorBasicAuthImpl(final String sparqlEndpoint, @Override public QueryExecution prepareSparqlQuery(final String query) { + if (released) { + throw new IllegalStateException("Cannot execute queries after releasing the connection"); + } return QueryExecutionFactory.sparqlService(queryEndpoint, query, client); } @Override public UpdateProcessor prepareSparqlUpdate(final String query) { + if (released) { + throw new IllegalStateException("Cannot execute queries after releasing the connection"); + } return UpdateExecutionFactory.createRemote( - UpdateFactory.create(query), - updateEndpoint, - client + UpdateFactory.create(query), + updateEndpoint, + client ); } + + @Override + public void release() { + try { + released = true; + client.close(); + } catch (IOException e) { + log.warn("Failed to close the HTTP client cleanly"); + log.debug("Failed to close the HTTP client cleanly", e); + } + } } diff --git a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorImpl.java b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorImpl.java index 78fe5dfde..96cf29281 100644 --- a/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorImpl.java +++ b/store/store-core/src/main/java/org/eclipse/lyo/store/internals/query/SparqlQueryExecutorImpl.java @@ -19,6 +19,8 @@ import org.apache.jena.update.UpdateExecutionFactory; import org.apache.jena.update.UpdateFactory; import org.apache.jena.update.UpdateProcessor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * SparqlQueryExecutorImpl is a SPARQL endpoint-based implementation of {@link JenaQueryExecutor}. @@ -28,6 +30,7 @@ * @since 0.14.0 */ public class SparqlQueryExecutorImpl implements JenaQueryExecutor { + private final Logger log = LoggerFactory.getLogger(SparqlQueryExecutorImpl.class); private final String queryEndpoint; private final String updateEndpoint; @@ -46,4 +49,9 @@ public QueryExecution prepareSparqlQuery(final String query) { public UpdateProcessor prepareSparqlUpdate(final String query) { return UpdateExecutionFactory.createRemote(UpdateFactory.create(query), updateEndpoint); } + + @Override + public void release() { + log.trace("NOP, there is nothing to release"); + } } diff --git a/store/store-core/src/test/java/org/eclipse/lyo/store/SparqlStoreImplTest.java b/store/store-core/src/test/java/org/eclipse/lyo/store/SparqlStoreImplTest.java index b33559559..9aa3a7688 100644 --- a/store/store-core/src/test/java/org/eclipse/lyo/store/SparqlStoreImplTest.java +++ b/store/store-core/src/test/java/org/eclipse/lyo/store/SparqlStoreImplTest.java @@ -14,10 +14,35 @@ * SPDX-License-Identifier: EPL-2.0 OR BSD-3-Clause */ +import com.google.common.base.Stopwatch; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.DatasetFactory; +import org.apache.jena.query.TxnType; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.tdb.TDBFactory; +import org.apache.jena.tdb2.TDB2Factory; +import org.eclipse.lyo.oslc4j.core.exception.OslcCoreApplicationException; +import org.eclipse.lyo.oslc4j.core.model.ServiceProvider; +import org.eclipse.lyo.oslc4j.provider.jena.JenaModelHelper; import org.eclipse.lyo.store.internals.SparqlStoreImpl; import org.eclipse.lyo.store.internals.query.DatasetQueryExecutorImpl; import org.junit.Before; import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import javax.xml.datatype.DatatypeConfigurationException; +import java.lang.reflect.InvocationTargetException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import static org.assertj.core.api.Assertions.*; +import static org.junit.Assert.*; /** * DatasetBuilderTest is . @@ -28,10 +53,16 @@ public class SparqlStoreImplTest extends StoreTestBase { private SparqlStoreImpl manager; + private Dataset dataset; @Before public void setUp() throws Exception { - manager = new SparqlStoreImpl(new DatasetQueryExecutorImpl()); + final Path tdbDir = Files.createTempDirectory("lyo_tdb_"); + System.out.println(tdbDir); + //FIXME make sure DatasetQueryExecutorImpl runs everything in a transaction +// dataset = TDB2Factory.connectDataset(tdbDir.toAbsolutePath().toString()); + dataset = TDBFactory.createDataset(tdbDir.toAbsolutePath().toString()); + manager = new SparqlStoreImpl(new DatasetQueryExecutorImpl(dataset)); } @Override @@ -41,5 +72,76 @@ protected SparqlStoreImpl buildStore() { @Override @Ignore("Not implemented yet") - public void testStoreKeySetReturnsCorrectKeys() {} + public void testStoreKeySetReturnsCorrectKeys() { + } + + @Test + public void datasetIsPersistentAndEmpty() { + assertTrue(TDB2Factory.isTDB2(dataset) || TDBFactory.isTDB1(dataset)); + try { + dataset.begin(TxnType.READ); + assertTrue(dataset.isEmpty()); + } finally { + dataset.end(); + } + } + + + @Test + public void storeBasicOps() { + final URI testNg = URI.create("urn:test:1"); + ServiceProvider sp = new ServiceProvider(); + sp.setIdentifier("123"); + sp.setCreated(new Date()); + try { + manager.putResources(testNg, Collections.singletonList(sp)); + final List providers = manager.getResources(testNg, ServiceProvider.class); + assertThat(providers).hasSize(1); + } catch (StoreAccessException | ModelUnmarshallingException e) { + fail("Store failed", e); + } + } + + @Test + public void testInsertionPerf() { + final List providers = genProviders(); + final Stopwatch stopwatch = Stopwatch.createStarted(); + for (int i = 0; i < 100; i++) { + final URI testNg = URI.create("urn:test:" + i); + try { + manager.putResources(testNg, providers); +// final List providers = manager.getResources(testNg, ServiceProvider.class); +// assertThat(providers).hasSize(1); + } catch (StoreAccessException e) { + fail("Store failed", e); + } + } + System.out.printf("100 named graphs persisted in %s", stopwatch.stop()); + } + + @Test + public void testInsertionPerfRaw() throws InvocationTargetException, DatatypeConfigurationException, OslcCoreApplicationException, IllegalAccessException { + final List providers = genProviders(); + final Model jenaModel = JenaModelHelper.createJenaModel(providers.toArray()); + final Stopwatch stopwatch = Stopwatch.createStarted(); + for (int i = 0; i < 100; i++) { + final URI testNg = URI.create("urn:test:" + i); + manager.insertJenaModel(testNg, jenaModel); + } + System.out.printf("100 named graphs persisted in %s", stopwatch.stop()); + } + + private List genProviders() { + final List providers = new ArrayList<>(); + for (int i = 0; i < 200; i++) { + ServiceProvider sp = new ServiceProvider(); + sp.setIdentifier(String.valueOf(i)); + sp.setCreated(new Date()); + sp.setDescription("Defaulting to no-operation (NOP) logger implementation"); + providers.add(sp); + } + return providers; + } + + }