From 42783434b26ee4a9b75c14106b11863f03f9592f Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Thu, 6 Oct 2016 18:59:07 -0400 Subject: [PATCH 1/4] RYA-142 updated for Fluo 1.0.0 release --- extras/indexing/pom.xml | 2 +- .../client/accumulo/AccumuloCreatePCJ.java | 4 +- .../client/accumulo/AccumuloDeletePCJ.java | 4 +- .../client/accumulo/FluoClientFactory.java | 8 +-- .../external/fluo/FluoPcjUpdater.java | 2 +- .../external/fluo/FluoPcjUpdaterSupplier.java | 6 +- extras/indexingExample/pom.xml | 2 +- .../src/main/java/RyaClientExample.java | 34 +++++----- extras/rya.console/pom.xml | 2 +- extras/rya.indexing.pcj/pom.xml | 4 +- .../pcj/fluo/api/CountStatements.java | 14 ++--- .../rya/indexing/pcj/fluo/api/CreatePcj.java | 6 +- .../rya/indexing/pcj/fluo/api/DeletePcj.java | 51 +++++++-------- .../indexing/pcj/fluo/api/GetPcjMetadata.java | 6 +- .../indexing/pcj/fluo/api/GetQueryReport.java | 28 ++++----- .../indexing/pcj/fluo/api/InsertTriples.java | 8 +-- .../indexing/pcj/fluo/api/ListQueryIds.java | 26 +++----- extras/rya.pcj.fluo/pcj.fluo.app/pom.xml | 11 ++-- .../indexing/pcj/fluo/app/BindingSetRow.java | 2 +- .../pcj/fluo/app/FilterResultUpdater.java | 10 +-- .../indexing/pcj/fluo/app/IncUpdateDAO.java | 58 ++++++----------- .../pcj/fluo/app/JoinResultUpdater.java | 62 ++++++++----------- .../rya/indexing/pcj/fluo/app/NodeType.java | 4 +- .../pcj/fluo/app/QueryResultUpdater.java | 10 +-- .../pcj/fluo/app/StringTypeLayer.java | 4 +- .../app/export/IncrementalResultExporter.java | 2 +- .../IncrementalResultExporterFactory.java | 2 +- .../app/export/rya/RyaExportParameters.java | 2 +- .../app/export/rya/RyaResultExporter.java | 4 +- .../export/rya/RyaResultExporterFactory.java | 6 +- .../fluo/app/observers/BindingSetUpdater.java | 14 ++--- .../fluo/app/observers/FilterObserver.java | 2 +- .../pcj/fluo/app/observers/JoinObserver.java | 2 +- .../app/observers/QueryResultObserver.java | 14 ++--- .../observers/StatementPatternObserver.java | 2 +- .../fluo/app/observers/TripleObserver.java | 45 ++++++-------- .../pcj/fluo/app/query/FluoQueryColumns.java | 4 +- .../fluo/app/query/FluoQueryMetadataDAO.java | 12 ++-- extras/rya.pcj.fluo/pcj.fluo.client/pom.xml | 2 +- .../pcj/fluo/client/PcjAdminClient.java | 6 +- .../fluo/client/PcjAdminClientCommand.java | 2 +- .../CountUnprocessedStatementsCommand.java | 2 +- .../client/command/ListQueriesCommand.java | 2 +- .../client/command/LoadTriplesCommand.java | 2 +- .../fluo/client/command/NewQueryCommand.java | 2 +- .../client/command/QueryReportCommand.java | 2 +- .../pcj/fluo/client/util/FluoLoader.java | 2 +- extras/rya.pcj.fluo/pcj.fluo.demo/pom.xml | 2 +- .../rya/indexing/pcj/fluo/demo/Demo.java | 4 +- .../indexing/pcj/fluo/demo/DemoDriver.java | 25 ++++---- .../fluo/demo/FluoAndHistoricPcjsDemo.java | 6 +- .../rya.pcj.fluo/pcj.fluo.integration/pom.xml | 4 +- .../apache/rya/indexing/pcj/fluo/ITBase.java | 29 +++++---- .../pcj/fluo/api/CountStatementsIT.java | 14 ++--- .../indexing/pcj/fluo/api/ListQueryIdsIT.java | 2 +- .../app/query/FluoQueryMetadataDAOIT.java | 4 +- .../pcj/fluo/integration/RyaExportIT.java | 2 +- .../RyaInputIncrementalUpdateIT.java | 4 +- pom.xml | 14 +++-- 59 files changed, 283 insertions(+), 328 deletions(-) diff --git a/extras/indexing/pom.xml b/extras/indexing/pom.xml index fb3042f87..422d9588f 100644 --- a/extras/indexing/pom.xml +++ b/extras/indexing/pom.xml @@ -109,7 +109,7 @@ test - io.fluo + org.apache.fluo fluo-mini test diff --git a/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloCreatePCJ.java b/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloCreatePCJ.java index 30be548f7..4cf093503 100644 --- a/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloCreatePCJ.java +++ b/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloCreatePCJ.java @@ -35,7 +35,7 @@ import com.google.common.base.Optional; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.accumulo.AccumuloRyaDAO; import mvm.rya.accumulo.instance.AccumuloRyaInstanceDetailsRepository; @@ -178,4 +178,4 @@ private static RyaSailRepository makeRyaRepository(final Connector connector, fi ryaRepo.initialize(); return ryaRepo; } -} \ No newline at end of file +} diff --git a/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloDeletePCJ.java b/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloDeletePCJ.java index d170fab4b..233a265c5 100644 --- a/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloDeletePCJ.java +++ b/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/AccumuloDeletePCJ.java @@ -32,7 +32,7 @@ import com.google.common.base.Optional; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.api.client.DeletePCJ; import mvm.rya.api.client.GetInstanceDetails; import mvm.rya.api.client.InstanceDoesNotExistException; @@ -127,4 +127,4 @@ private void stopUpdatingPCJ(final String fluoAppName, final String pcjId) { // Delete the PCJ from the Fluo App. new DeletePcj(1000).deletePcj(fluoClient, pcjId); } -} \ No newline at end of file +} diff --git a/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/FluoClientFactory.java b/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/FluoClientFactory.java index b80abfa7c..0c6c2f164 100644 --- a/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/FluoClientFactory.java +++ b/extras/indexing/src/main/java/mvm/rya/api/client/accumulo/FluoClientFactory.java @@ -22,9 +22,9 @@ import javax.annotation.ParametersAreNonnullByDefault; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.config.FluoConfiguration; /** * Creates {@link FluoClient}s that are connected to a specific Fluo Application. @@ -69,4 +69,4 @@ public FluoClient connect( // Connect the client. return FluoFactory.newClient(fluoConfig); } -} \ No newline at end of file +} diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdater.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdater.java index 901bd611c..f53727d8f 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdater.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdater.java @@ -30,7 +30,7 @@ import com.google.common.base.Optional; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.api.domain.RyaStatement; /** diff --git a/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java b/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java index 61de07870..c0895ba75 100644 --- a/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java +++ b/extras/indexing/src/main/java/mvm/rya/indexing/external/fluo/FluoPcjUpdaterSupplier.java @@ -26,9 +26,9 @@ import static mvm.rya.indexing.external.fluo.FluoPcjUpdaterConfig.ACCUMULO_ZOOKEEPERS; import static mvm.rya.indexing.external.fluo.FluoPcjUpdaterConfig.FLUO_APP_NAME; import static mvm.rya.indexing.external.fluo.FluoPcjUpdaterConfig.STATEMENT_VISIBILITY; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.config.FluoConfiguration; import javax.annotation.ParametersAreNonnullByDefault; diff --git a/extras/indexingExample/pom.xml b/extras/indexingExample/pom.xml index 58689de0a..8117f47d5 100644 --- a/extras/indexingExample/pom.xml +++ b/extras/indexingExample/pom.xml @@ -74,7 +74,7 @@ under the License. - io.fluo + org.apache.fluo fluo-mini diff --git a/extras/indexingExample/src/main/java/RyaClientExample.java b/extras/indexingExample/src/main/java/RyaClientExample.java index 7d1d3dc26..bee44e12e 100644 --- a/extras/indexingExample/src/main/java/RyaClientExample.java +++ b/extras/indexingExample/src/main/java/RyaClientExample.java @@ -53,13 +53,13 @@ import com.google.common.io.Files; import info.aduna.iteration.CloseableIteration; -import io.fluo.api.client.FluoAdmin; -import io.fluo.api.client.FluoAdmin.AlreadyInitializedException; -import io.fluo.api.client.FluoAdmin.TableExistsException; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.config.FluoConfiguration; -import io.fluo.api.config.ObserverConfiguration; -import io.fluo.api.mini.MiniFluo; +import org.apache.fluo.api.client.FluoAdmin; +import org.apache.fluo.api.client.FluoAdmin.AlreadyInitializedException; +import org.apache.fluo.api.client.FluoAdmin.TableExistsException; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.config.ObserverSpecification; +import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.client.Install.InstallConfiguration; import mvm.rya.api.client.RyaClient; @@ -242,17 +242,14 @@ private static String[] prettyFormatSparql(final String sparql) throws Exception private static MiniFluo makeMiniFluo(final String username, final String password, final String instanceName, final String zookeepers, final String fluoAppName) throws AlreadyInitializedException, TableExistsException { // Setup the observers that will be used by the Fluo PCJ Application. - final List observers = new ArrayList<>(); - observers.add(new ObserverConfiguration(TripleObserver.class.getName())); - observers.add(new ObserverConfiguration(StatementPatternObserver.class.getName())); - observers.add(new ObserverConfiguration(JoinObserver.class.getName())); - observers.add(new ObserverConfiguration(FilterObserver.class.getName())); + final List observers = new ArrayList<>(); + observers.add(new ObserverSpecification(TripleObserver.class.getName())); + observers.add(new ObserverSpecification(StatementPatternObserver.class.getName())); + observers.add(new ObserverSpecification(JoinObserver.class.getName())); + observers.add(new ObserverSpecification(FilterObserver.class.getName())); // Provide export parameters child test classes may provide to the // export observer. - final ObserverConfiguration exportObserverConfig = new ObserverConfiguration( - QueryResultObserver.class.getName()); - final HashMap params = new HashMap<>(); final RyaExportParameters ryaParams = new RyaExportParameters(params); ryaParams.setExportToRya(true); @@ -262,7 +259,8 @@ private static MiniFluo makeMiniFluo(final String username, final String passwor ryaParams.setExporterPassword(password); ryaParams.setRyaInstanceName(fluoAppName); - exportObserverConfig.setParameters(params); + final ObserverSpecification exportObserverConfig = new ObserverSpecification( + QueryResultObserver.class.getName(), params); observers.add(exportObserverConfig); // Configure how the mini fluo will run. @@ -280,7 +278,7 @@ private static MiniFluo makeMiniFluo(final String username, final String passwor config.addObservers(observers); FluoFactory.newAdmin(config).initialize( - new FluoAdmin.InitOpts().setClearTable(true).setClearZookeeper(true) ); + new FluoAdmin.InitializationOptions().setClearTable(true).setClearZookeeper(true) ); return FluoFactory.newMiniFluo(config); } -} \ No newline at end of file +} diff --git a/extras/rya.console/pom.xml b/extras/rya.console/pom.xml index 9513c59cd..ea5c2bbc4 100644 --- a/extras/rya.console/pom.xml +++ b/extras/rya.console/pom.xml @@ -84,7 +84,7 @@ test - io.fluo + org.apache.fluo fluo-mini test diff --git a/extras/rya.indexing.pcj/pom.xml b/extras/rya.indexing.pcj/pom.xml index c90ded362..3d9bf960a 100644 --- a/extras/rya.indexing.pcj/pom.xml +++ b/extras/rya.indexing.pcj/pom.xml @@ -56,7 +56,7 @@ under the License. - io.fluo + org.apache.fluo fluo-api @@ -91,4 +91,4 @@ under the License. test-jar - \ No newline at end of file + diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CountStatements.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CountStatements.java index 326f80746..9b7e0c576 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CountStatements.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CountStatements.java @@ -21,13 +21,13 @@ import static com.google.common.base.Preconditions.checkNotNull; import java.math.BigInteger; +import java.util.Iterator; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.Snapshot; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.iterator.RowIterator; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.scanner.ColumnScanner; /** * Counts the number of RDF Statements that have been loaded into the Fluo app @@ -48,10 +48,8 @@ public BigInteger countStatements(final FluoClient fluo) { try(Snapshot sx = fluo.newSnapshot()) { // Limit the scan to the Triples binding set column. - final ScannerConfiguration scanConfig = new ScannerConfiguration(); - scanConfig.fetchColumn(FluoQueryColumns.TRIPLES.getFamily(), FluoQueryColumns.TRIPLES.getQualifier()); - - final RowIterator rows = sx.get(scanConfig); + final Iterator rows = sx.scanner().fetch(FluoQueryColumns.TRIPLES).byRow().build().iterator(); + BigInteger count = BigInteger.valueOf(0L); while(rows.hasNext()) { rows.next(); diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java index 29d328d79..943a0220b 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java @@ -54,8 +54,8 @@ import org.openrdf.sail.SailException; import info.aduna.iteration.CloseableIteration; -import io.fluo.api.client.FluoClient; -import io.fluo.api.types.TypedTransaction; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.recipes.core.types.TypedTransaction; /** * Sets up a new Pre Computed Join (PCJ) in Fluo from a SPARQL query. @@ -229,4 +229,4 @@ private static void writeBatch(final FluoClient fluo, final StatementPatternMeta tx.commit(); } } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java index df59cb56f..e1a9b8e3b 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java @@ -22,6 +22,7 @@ import static java.util.Objects.requireNonNull; import java.util.ArrayList; +import java.util.Iterator; import java.util.List; import javax.annotation.ParametersAreNonnullByDefault; @@ -35,14 +36,14 @@ import org.apache.rya.indexing.pcj.fluo.app.query.QueryMetadata; import org.openrdf.query.BindingSet; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.Transaction; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.data.Span; -import io.fluo.api.iterator.RowIterator; -import io.fluo.api.types.TypedTransaction; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.Transaction; +import org.apache.fluo.api.client.scanner.CellScanner; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.api.data.RowColumnValue; +import org.apache.fluo.api.data.Span; +import org.apache.fluo.recipes.core.types.TypedTransaction; /** * Deletes a Pre-computed Join (PCJ) from Fluo. @@ -239,34 +240,30 @@ private void deleteData(final FluoClient client, final String nodeId) { } } - private RowIterator getIterator(final Transaction tx, final String nodeId, final Column column) { + private CellScanner getIterator(final Transaction tx, final String nodeId, final Column column) { requireNonNull(tx); requireNonNull(nodeId); requireNonNull(column); - ScannerConfiguration sc1 = new ScannerConfiguration(); - sc1.fetchColumn(column.getFamily(), column.getQualifier()); - sc1.setSpan(Span.prefix(Bytes.of(nodeId))); - return tx.get(sc1); + return tx.scanner().fetch(column).over(Span.prefix(nodeId)).build(); } - private boolean deleteDataBatch(final Transaction tx, final RowIterator iter, final Column column) { + private boolean deleteDataBatch(final Transaction tx, final CellScanner scanner, final Column column) { requireNonNull(tx); - requireNonNull(iter); + requireNonNull(scanner); requireNonNull(column); - try (final TypedTransaction typeTx = new StringTypeLayer().wrap(tx)) { - int count = 0; - while (iter.hasNext() && count < batchSize) { - final Bytes row = iter.next().getKey(); - count++; - tx.delete(row, column); - } - - final boolean hasNext = iter.hasNext(); - tx.commit(); - return hasNext; + int count = 0; + Iterator iter = scanner.iterator(); + while (iter.hasNext() && count < batchSize) { + final Bytes row = iter.next().getRow(); + count++; + tx.delete(row, column); } + + final boolean hasNext = iter.hasNext(); + tx.commit(); + return hasNext; } private String getQueryIdFromPcjId(final Transaction tx, final String pcjId) { @@ -284,4 +281,4 @@ private String getSparqlFromQueryId(final Transaction tx, final String queryId) final QueryMetadata metadata = dao.readQueryMetadata(tx, queryId); return metadata.getSparql(); } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java index 7a0a95324..d8c800e66 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java @@ -30,9 +30,9 @@ import org.apache.rya.indexing.pcj.storage.PcjMetadata; import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage; -import io.fluo.api.client.FluoClient; -import io.fluo.api.data.Bytes; -import io.fluo.api.types.TypedSnapshot; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.recipes.core.types.TypedSnapshot; /** * Get {@link PcjMetadata} for queries that are managed by the Fluo app. diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReport.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReport.java index 2db7f3da3..e1e8566fa 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReport.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetQueryReport.java @@ -38,13 +38,13 @@ import com.google.common.collect.ImmutableMap; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.Snapshot; -import io.fluo.api.client.SnapshotBase; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Column; -import io.fluo.api.data.Span; -import io.fluo.api.iterator.RowIterator; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.SnapshotBase; +import org.apache.fluo.api.client.scanner.ColumnScanner; +import org.apache.fluo.api.client.scanner.RowScanner; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.api.data.Span; /** * Get a reports that indicates how many binding sets have been emitted for @@ -130,17 +130,13 @@ private BigInteger countBindingSets(final SnapshotBase sx, final String nodeId, checkNotNull(bindingSetColumn); // Limit the scan to the binding set column and node id. - final ScannerConfiguration scanConfig = new ScannerConfiguration(); - scanConfig.fetchColumn(bindingSetColumn.getFamily(), bindingSetColumn.getQualifier()); - scanConfig.setSpan( Span.prefix(nodeId) ); + final RowScanner rows = sx.scanner().over(Span.prefix(nodeId)).fetch(bindingSetColumn).byRow().build(); - final RowIterator rows = sx.get(scanConfig); BigInteger count = BigInteger.valueOf(0L); - while(rows.hasNext()) { - rows.next(); - count = count.add( BigInteger.ONE ); - } - + for (ColumnScanner columns : rows) { + count = count.add( BigInteger.ONE ); + } + return count; } diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java index 2a53a0ebd..1f4538829 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java @@ -30,10 +30,10 @@ import com.google.common.base.Optional; -import io.fluo.api.client.FluoClient; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; -import io.fluo.api.types.TypedTransaction; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; +import org.apache.fluo.recipes.core.types.TypedTransaction; import mvm.rya.api.RdfCloudTripleStoreConstants.TABLE_LAYOUT; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.resolver.triple.TripleRow; diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java index a85bf56b6..3913e410a 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java @@ -23,17 +23,14 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map.Entry; import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; -import io.fluo.api.client.FluoClient; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Bytes; -import io.fluo.api.iterator.ColumnIterator; -import io.fluo.api.iterator.RowIterator; -import io.fluo.api.types.TypedSnapshot; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.scanner.CellScanner; +import org.apache.fluo.api.data.RowColumnValue; +import org.apache.fluo.recipes.core.types.TypedSnapshot; /** * Finds all queries that are being managed by this instance of Fluo that @@ -56,17 +53,12 @@ public List listQueryIds(final FluoClient fluo) { try(TypedSnapshot snap = new StringTypeLayer().wrap( fluo.newSnapshot() )) { // Create an iterator that iterates over the QUERY_ID column. - final ScannerConfiguration scanConfig = new ScannerConfiguration(); - scanConfig.fetchColumn(FluoQueryColumns.QUERY_ID.getFamily(), FluoQueryColumns.QUERY_ID.getQualifier()); - final RowIterator rows = snap.get(scanConfig); + final CellScanner cellScanner = snap.scanner().fetch( FluoQueryColumns.QUERY_ID).build(); - // Fetch the Query IDs that is stored in the Fluo table. - while(rows.hasNext()) { - final Entry entry = rows.next(); - final Bytes sparql = entry.getKey(); - final String queryId = snap.get(sparql, FluoQueryColumns.QUERY_ID).toString(); - queryIds.add(queryId); - } + for (RowColumnValue rcv : cellScanner) { + queryIds.add(rcv.getsValue()); + //TODO this was doing a snap.get that seemed unnecessary + } } // Sort them alphabetically. diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml b/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml index 9c7f672da..45ea9ce28 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml +++ b/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml @@ -52,11 +52,15 @@ under the License. - io.fluo + org.apache.fluo fluo-api - io.fluo + org.apache.fluo + fluo-recipes-core + + + org.apache.fluo fluo-core @@ -65,7 +69,6 @@ under the License. - junit @@ -97,4 +100,4 @@ under the License. - \ No newline at end of file + diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/BindingSetRow.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/BindingSetRow.java index 85f1b1fcb..fa0ea70c6 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/BindingSetRow.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/BindingSetRow.java @@ -24,7 +24,7 @@ import javax.annotation.ParametersAreNonnullByDefault; import javax.annotation.concurrent.Immutable; -import io.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Bytes; /** * The values of an Accumulo Row ID for a row that stores a Binding set for diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java index 8583eb5ff..328c6537c 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java @@ -49,11 +49,11 @@ import com.google.common.base.Optional; import info.aduna.iteration.CloseableIteration; -import io.fluo.api.client.TransactionBase; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; +import org.apache.fluo.api.client.TransactionBase; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; /** * Updates the results of a Filter node when its child has added a new Binding diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java index 5bc9d0a3d..7ccfeff6e 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java @@ -23,16 +23,13 @@ import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.TYPE_DELIM; import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.URI_TYPE; -import java.util.Map.Entry; - -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.Snapshot; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.iterator.ColumnIterator; -import io.fluo.api.iterator.RowIterator; -import io.fluo.api.types.TypedTransaction; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.scanner.CellScanner; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.api.data.RowColumnValue; +import org.apache.fluo.recipes.core.types.TypedTransaction; import mvm.rya.api.RdfCloudTripleStoreConstants.TABLE_LAYOUT; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.resolver.triple.TripleRow; @@ -106,15 +103,10 @@ public static void addRow(final TypedTransaction tx, final String row, final Col */ public static void printTriples(final FluoClient fluoClient) throws Exception { try (Snapshot snapshot = fluoClient.newSnapshot()) { - final ScannerConfiguration scanConfig = new ScannerConfiguration(); - scanConfig.fetchColumn(Bytes.of("triples"), Bytes.of("SPO")); - - final RowIterator rowIter = snapshot.get(scanConfig); - - while (rowIter.hasNext()) { - final Entry row = rowIter.next(); - System.out.println("Triple: " + row.getKey().toString()); - } + CellScanner cscanner = snapshot.scanner().fetch(new Column("triples", "SPO")).build(); + for (RowColumnValue rcv : cscanner) { + System.out.println("Triple: "+rcv.getsRow()); + } } } @@ -179,25 +171,13 @@ public static void printAll(final FluoClient fluoClient) throws Exception { "--Column Qual--", "--Value--"); // Use try with resource to ensure snapshot is closed. try (Snapshot snapshot = fluoClient.newSnapshot()) { - final ScannerConfiguration scanConfig = new ScannerConfiguration(); - // scanConfig.setSpan(Span.prefix("word:")); - - final RowIterator rowIter = snapshot.get(scanConfig); - - while (rowIter.hasNext()) { - final Entry row = rowIter.next(); - final ColumnIterator colIter = row.getValue(); - while (colIter.hasNext()) { - final Entry column = colIter.next(); - // System.out.println(row.getKey() + " " + - // column.getKey().getFamily()+ " " + - // column.getKey().getQualifier() ); - System.out.format(FORMAT, to_String(row.getKey()), - to_String(column.getKey().getFamily()), - to_String(column.getKey().getQualifier()), - to_String(column.getValue())); - } - } + CellScanner cscanner = snapshot.scanner().build(); + for (RowColumnValue rcv : cscanner) { + System.out.format(FORMAT, to_String(rcv.getRow()), + to_String(rcv.getColumn().getFamily()), + to_String(rcv.getColumn().getQualifier()), + to_String(rcv.getValue())); + } } } @@ -229,4 +209,4 @@ static String to_String(final byte[] bytes) { } return sb.toString(); } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java index dfa5b626d..5ac69b013 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java @@ -25,7 +25,6 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.Map.Entry; import java.util.Set; import javax.annotation.ParametersAreNonnullByDefault; @@ -47,15 +46,15 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import io.fluo.api.client.TransactionBase; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.data.Span; -import io.fluo.api.iterator.ColumnIterator; -import io.fluo.api.iterator.RowIterator; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; +import org.apache.fluo.api.client.TransactionBase; +import org.apache.fluo.api.client.scanner.ColumnScanner; +import org.apache.fluo.api.client.scanner.RowScanner; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.api.data.ColumnValue; +import org.apache.fluo.api.data.Span; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; /** * Updates the results of a Join node when one of its children has added a @@ -175,12 +174,9 @@ private FluoTableIterator makeSiblingScanIterator(final String childId, final Bi // and inserted into the Join's results. It's possible that none of these // results will be new Join results if they have already been created in // earlier iterations of this algorithm. - final ScannerConfiguration scanConfig = new ScannerConfiguration(); - scanConfig.setSpan(Span.prefix(siblingScanPrefix)); - setScanColumnFamily(scanConfig, siblingId); - final RowIterator ri = tx.get(scanConfig); - return new FluoTableIterator(ri, siblingVarOrder); + final RowScanner rs = tx.scanner().over(Span.prefix(siblingScanPrefix)).fetch(getScanColumnFamily(siblingId)).byRow().build(); + return new FluoTableIterator(rs, siblingVarOrder); } @@ -247,15 +243,12 @@ public List getCommonVars(final VariableOrder vars1, final VariableOrder } /** - * Update a {@link ScannerConfiguration} to use the sibling node's binding - * set column for its scan. The column that will be used is determined by the - * node's {@link NodeType}. + * Return the sibling node's binding set column to use for a scan. The column + * that will be used is determined by the node's {@link NodeType}. * - * @param sc - The scan configuration that will be updated. (not null) * @param siblingId - The Node ID of the sibling. (not null) */ - private static void setScanColumnFamily(final ScannerConfiguration sc, final String siblingId) { - checkNotNull(sc); + private static Column getScanColumnFamily(final String siblingId) { checkNotNull(siblingId); // Determine which type of binding set the sibling is. @@ -279,7 +272,8 @@ private static void setScanColumnFamily(final ScannerConfiguration sc, final Str default: throw new IllegalArgumentException("The child node's sibling is not of type StatementPattern, Join, Left Join, or Filter."); } - sc.fetchColumn(column.getFamily(), column.getQualifier()); + + return column; } /** @@ -462,7 +456,7 @@ private static final class FluoTableIterator implements Iterator rows; private final VariableOrder varOrder; /** @@ -472,8 +466,8 @@ private static final class FluoTableIterator implements Iterator entry = columns.next(); - if(BINDING_SET_COLUMNS.contains(entry.getKey())) { - final String bindingSetString = entry.getValue().toString(); - return (VisibilityBindingSet) valueConverter.convert(bindingSetString, varOrder); - } - } + final ColumnScanner columns = rows.next(); + + for (ColumnValue cv : columns) { + if(BINDING_SET_COLUMNS.contains(cv.getColumn())) { + final String bindingSetString = cv.getsValue(); + return (VisibilityBindingSet) valueConverter.convert(bindingSetString, varOrder); + } + } throw new RuntimeException("Row did not containing a Binding Set."); } diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java index c6ad31e3c..0a5ecc10c 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/NodeType.java @@ -32,7 +32,7 @@ import com.google.common.base.Optional; -import io.fluo.api.data.Column; +import org.apache.fluo.api.data.Column; /** * Represents the different types of nodes that a Query may have. @@ -99,4 +99,4 @@ public static Optional fromNodeId(final String nodeId) { return Optional.fromNullable(type); } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java index 0c5798503..41f902598 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java @@ -32,11 +32,11 @@ import org.openrdf.query.Binding; import org.openrdf.query.impl.MapBindingSet; -import io.fluo.api.client.TransactionBase; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; +import org.apache.fluo.api.client.TransactionBase; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; /** * Updates the results of a Query node when one of its children has added a diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java index 35fcf52f7..aecb434c4 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java @@ -18,8 +18,8 @@ */ package org.apache.rya.indexing.pcj.fluo.app; -import io.fluo.api.types.StringEncoder; -import io.fluo.api.types.TypeLayer; +import org.apache.fluo.recipes.core.types.StringEncoder; +import org.apache.fluo.recipes.core.types.TypeLayer; public class StringTypeLayer extends TypeLayer { diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java index d93bae290..a7f016d8e 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java @@ -22,7 +22,7 @@ import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet; -import io.fluo.api.types.TypedTransactionBase; +import org.apache.fluo.recipes.core.types.TypedTransactionBase; /** * Exports a single Binding Set that is a new result for a SPARQL query to some diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporterFactory.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporterFactory.java index aae42ef30..70520696b 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporterFactory.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporterFactory.java @@ -22,7 +22,7 @@ import com.google.common.base.Optional; -import io.fluo.api.observer.Observer.Context; +import org.apache.fluo.api.observer.Observer.Context; /** * Builds instances of {@link IncrementalResultExporter} using the provided diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaExportParameters.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaExportParameters.java index 6f71a48d3..18af878d5 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaExportParameters.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaExportParameters.java @@ -27,7 +27,7 @@ import com.google.common.base.Optional; -import io.fluo.api.observer.Observer; +import org.apache.fluo.api.observer.Observer; /** * Provides read/write functions to the parameters map that is passed into an diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java index 11245c041..27530f006 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java @@ -28,8 +28,8 @@ import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException; import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet; -import io.fluo.api.data.Bytes; -import io.fluo.api.types.TypedTransactionBase; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.recipes.core.types.TypedTransactionBase; /** * Incrementally exports SPARQL query results to Accumulo PCJ tables as they are defined by Rya. diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporterFactory.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporterFactory.java index dfbb91088..c695272f9 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporterFactory.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporterFactory.java @@ -33,7 +33,7 @@ import com.google.common.base.Optional; -import io.fluo.api.observer.Observer.Context; +import org.apache.fluo.api.observer.Observer.Context; /** * Creates instances of {@link RyaResultExporter}. @@ -45,7 +45,7 @@ public Optional build(final Context context) throws I checkNotNull(context); // Wrap the context's parameters for parsing. - final RyaExportParameters params = new RyaExportParameters( context.getParameters() ); + final RyaExportParameters params = new RyaExportParameters( context.getObserverConfiguration().toMap() ); if(params.isExportToRya()) { // Setup Zookeeper connection info. @@ -74,4 +74,4 @@ public Optional build(final Context context) throws I return Optional.absent(); } } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java index 06fca0abf..e344b0ab9 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java @@ -34,13 +34,13 @@ import org.apache.rya.indexing.pcj.storage.accumulo.BindingSetConverter.BindingSetConversionException; import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet; -import io.fluo.api.client.TransactionBase; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; -import io.fluo.api.types.TypedObserver; -import io.fluo.api.types.TypedTransactionBase; +import org.apache.fluo.api.client.TransactionBase; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; +import org.apache.fluo.recipes.core.types.TypedObserver; +import org.apache.fluo.recipes.core.types.TypedTransactionBase; /** * Notified when the results of a node have been updated to include a new Binding diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/FilterObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/FilterObserver.java index ea1f3a067..36af89841 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/FilterObserver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/FilterObserver.java @@ -29,7 +29,7 @@ import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSetStringConverter; import org.openrdf.query.BindingSet; -import io.fluo.api.client.TransactionBase; +import org.apache.fluo.api.client.TransactionBase; /** * Notified when the results of a Filter have been updated to include a new diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/JoinObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/JoinObserver.java index dbde8859d..693309630 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/JoinObserver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/JoinObserver.java @@ -29,7 +29,7 @@ import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSetStringConverter; import org.openrdf.query.BindingSet; -import io.fluo.api.client.TransactionBase; +import org.apache.fluo.api.client.TransactionBase; /** * Notified when the results of a Join have been updated to include a new diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java index 2f7a1eafd..638b1fcc9 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java @@ -39,12 +39,12 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; -import io.fluo.api.types.TypedObserver; -import io.fluo.api.types.TypedTransactionBase; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; +import org.apache.fluo.recipes.core.types.TypedObserver; +import org.apache.fluo.recipes.core.types.TypedTransactionBase; import mvm.rya.accumulo.utils.VisibilitySimplifier; /** @@ -138,4 +138,4 @@ public void process(final TypedTransactionBase tx, final Bytes row, final Column } } } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/StatementPatternObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/StatementPatternObserver.java index 46943716d..59566349e 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/StatementPatternObserver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/StatementPatternObserver.java @@ -29,7 +29,7 @@ import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSetStringConverter; import org.openrdf.query.BindingSet; -import io.fluo.api.client.TransactionBase; +import org.apache.fluo.api.client.TransactionBase; /** * Notified when the results of a Statement Pattern have been updated to include diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java index 4feb752ed..31a4c29e1 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java @@ -24,7 +24,6 @@ import static org.apache.rya.indexing.pcj.fluo.app.IncrementalUpdateConstants.VAR_DELIM; import java.util.Map; -import java.util.Map.Entry; import org.apache.rya.indexing.pcj.fluo.app.IncUpdateDAO; import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; @@ -37,16 +36,16 @@ import com.google.common.collect.Maps; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.data.Span; -import io.fluo.api.iterator.ColumnIterator; -import io.fluo.api.iterator.RowIterator; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; -import io.fluo.api.types.TypedObserver; -import io.fluo.api.types.TypedTransactionBase; +import org.apache.fluo.api.client.scanner.ColumnScanner; +import org.apache.fluo.api.client.scanner.RowScanner; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.api.data.ColumnValue; +import org.apache.fluo.api.data.Span; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; +import org.apache.fluo.recipes.core.types.TypedObserver; +import org.apache.fluo.recipes.core.types.TypedTransactionBase; /** * An observer that matches new Triples to the Statement Patterns that are part @@ -79,23 +78,19 @@ public void process(final TypedTransactionBase tx, final Bytes row, final Column } //get variable metadata for all SP in table - final ScannerConfiguration sc1 = new ScannerConfiguration(); - sc1.fetchColumn(FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER.getFamily(), FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER.getQualifier()); - sc1.setSpan(Span.prefix(SP_PREFIX)); + RowScanner rscanner = tx.scanner().over(Span.prefix(SP_PREFIX)).fetch(FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER).byRow().build(); + //see if triple matches conditions of any of the SP - final RowIterator ri = tx.get(sc1); - while(ri.hasNext()) { - final Entry next = ri.next(); - final ColumnIterator ci = next.getValue(); - final String spID = next.getKey().toString(); + for (ColumnScanner colScanner : rscanner) { + final String spID = colScanner.getsRow(); final StatementPatternMetadata spMetadata = QUERY_DAO.readStatementPatternMetadata(tx, spID); final String pattern = spMetadata.getStatementPattern(); - - while(ci.hasNext()) { - final String varOrders = ci.next().getValue().toString(); + + for (ColumnValue cv : colScanner) { + final String varOrders = cv.getsValue(); final VariableOrder varOrder = new VariableOrder(varOrders); final String bindingSetString = getBindingSet(triple, pattern, varOrders); @@ -107,8 +102,8 @@ public void process(final TypedTransactionBase tx, final Bytes row, final Column final String valueString = CONVERTER.convert(bindingSet, varOrder); tx.mutate().row(spID + NODEID_BS_DELIM + bindingSetString).col(FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET).set(valueString); } - } - } + } + } // Once the triple has been handled, it may be deleted. tx.delete(row, column); @@ -158,4 +153,4 @@ private static String getBindingSet(final String triple, final String spID, fina return bindingSet; } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java index e90496a18..13743f48f 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryColumns.java @@ -25,7 +25,7 @@ import javax.annotation.ParametersAreNonnullByDefault; -import io.fluo.api.data.Column; +import org.apache.fluo.api.data.Column; /** * Holds {@link Column} objects that represent where each piece of metadata is stored. @@ -223,4 +223,4 @@ public List columns() { return columns; } } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java index e33ab0773..a955a538c 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java @@ -30,12 +30,12 @@ import com.google.common.collect.Sets; -import io.fluo.api.client.SnapshotBase; -import io.fluo.api.client.TransactionBase; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Column; -import io.fluo.api.types.Encoder; -import io.fluo.api.types.StringEncoder; +import org.apache.fluo.api.client.SnapshotBase; +import org.apache.fluo.api.client.TransactionBase; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Column; +import org.apache.fluo.recipes.core.types.Encoder; +import org.apache.fluo.recipes.core.types.StringEncoder; /** * Reads and writes {@link FluoQuery} instances and their components to/from diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/pom.xml b/extras/rya.pcj.fluo/pcj.fluo.client/pom.xml index 874615484..a02aa6cb1 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/pom.xml +++ b/extras/rya.pcj.fluo/pcj.fluo.client/pom.xml @@ -52,7 +52,7 @@ under the License. - io.fluo + org.apache.fluo fluo-core diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClient.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClient.java index 59177f6e9..1e70d6ed5 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClient.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClient.java @@ -54,9 +54,9 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.config.FluoConfiguration; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.accumulo.AccumuloRyaDAO; import mvm.rya.rdftriplestore.RdfCloudTripleStore; diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClientCommand.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClientCommand.java index e1185d441..afe6b9f49 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClientCommand.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/PcjAdminClientCommand.java @@ -22,7 +22,7 @@ import org.apache.accumulo.core.client.Connector; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.rdftriplestore.RyaSailRepository; /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/CountUnprocessedStatementsCommand.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/CountUnprocessedStatementsCommand.java index 7fefb28f4..151501da5 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/CountUnprocessedStatementsCommand.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/CountUnprocessedStatementsCommand.java @@ -27,7 +27,7 @@ import org.apache.rya.indexing.pcj.fluo.api.CountStatements; import org.apache.rya.indexing.pcj.fluo.client.PcjAdminClientCommand; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.rdftriplestore.RyaSailRepository; /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/ListQueriesCommand.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/ListQueriesCommand.java index 3e844991e..20ddf1712 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/ListQueriesCommand.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/ListQueriesCommand.java @@ -41,7 +41,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.rdftriplestore.RyaSailRepository; /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/LoadTriplesCommand.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/LoadTriplesCommand.java index c24eb6644..2562378c1 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/LoadTriplesCommand.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/LoadTriplesCommand.java @@ -45,7 +45,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.rdftriplestore.RyaSailRepository; /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java index ad6675720..0f553d69b 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/NewQueryCommand.java @@ -45,7 +45,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.rdftriplestore.RyaSailRepository; /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/QueryReportCommand.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/QueryReportCommand.java index 9a8c7f745..5ae29b631 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/QueryReportCommand.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/command/QueryReportCommand.java @@ -32,7 +32,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.rdftriplestore.RyaSailRepository; /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/FluoLoader.java b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/FluoLoader.java index 7be539a22..c5ec7cc92 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/FluoLoader.java +++ b/extras/rya.pcj.fluo/pcj.fluo.client/src/main/java/org/apache/rya/indexing/pcj/fluo/client/util/FluoLoader.java @@ -32,7 +32,7 @@ import com.google.common.base.Optional; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.resolver.RdfToRyaConversions; diff --git a/extras/rya.pcj.fluo/pcj.fluo.demo/pom.xml b/extras/rya.pcj.fluo/pcj.fluo.demo/pom.xml index 7396cee92..7deea2d87 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.demo/pom.xml +++ b/extras/rya.pcj.fluo/pcj.fluo.demo/pom.xml @@ -53,7 +53,7 @@ under the License. sesame-queryrender - io.fluo + org.apache.fluo fluo-mini diff --git a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/Demo.java b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/Demo.java index f1ec3dd96..f02b0994d 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/Demo.java +++ b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/Demo.java @@ -22,8 +22,8 @@ import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.openrdf.repository.RepositoryConnection; -import io.fluo.api.client.FluoClient; -import io.fluo.api.mini.MiniFluo; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.rdftriplestore.RyaSailRepository; /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/DemoDriver.java b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/DemoDriver.java index 1d37831d8..b1a1c917c 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/DemoDriver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/DemoDriver.java @@ -52,11 +52,11 @@ import com.google.common.base.Optional; import com.google.common.io.Files; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.config.FluoConfiguration; -import io.fluo.api.config.ObserverConfiguration; -import io.fluo.api.mini.MiniFluo; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.config.ObserverSpecification; +import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.accumulo.AccumuloRyaDAO; import mvm.rya.accumulo.instance.AccumuloRyaInstanceDetailsRepository; @@ -297,11 +297,11 @@ private static MiniFluo startMiniFluo() { final File miniDataDir = Files.createTempDir(); // Setup the observers that will be used by the Fluo PCJ Application. - final List observers = new ArrayList<>(); - observers.add(new ObserverConfiguration(TripleObserver.class.getName())); - observers.add(new ObserverConfiguration(StatementPatternObserver.class.getName())); - observers.add(new ObserverConfiguration(JoinObserver.class.getName())); - observers.add(new ObserverConfiguration(FilterObserver.class.getName())); + final List observers = new ArrayList<>(); + observers.add(new ObserverSpecification(TripleObserver.class.getName())); + observers.add(new ObserverSpecification(StatementPatternObserver.class.getName())); + observers.add(new ObserverSpecification(JoinObserver.class.getName())); + observers.add(new ObserverSpecification(FilterObserver.class.getName())); // Provide export parameters child test classes may provide to the export observer. final HashMap exportParams = new HashMap<>(); @@ -313,8 +313,7 @@ private static MiniFluo startMiniFluo() { ryaParams.setExporterPassword("password"); ryaParams.setRyaInstanceName("demo_"); - final ObserverConfiguration exportObserverConfig = new ObserverConfiguration(QueryResultObserver.class.getName()); - exportObserverConfig.setParameters( exportParams ); + final ObserverSpecification exportObserverConfig = new ObserverSpecification(QueryResultObserver.class.getName(), exportParams); observers.add(exportObserverConfig); // Configure how the mini fluo will run. @@ -326,4 +325,4 @@ private static MiniFluo startMiniFluo() { final MiniFluo miniFluo = FluoFactory.newMiniFluo(config); return miniFluo; } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java index 820796648..cb4420d76 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java +++ b/extras/rya.pcj.fluo/pcj.fluo.demo/src/main/java/org/apache/rya/indexing/pcj/fluo/demo/FluoAndHistoricPcjsDemo.java @@ -45,8 +45,8 @@ import com.google.common.base.Optional; import com.google.common.collect.Sets; -import io.fluo.api.client.FluoClient; -import io.fluo.api.mini.MiniFluo; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.domain.RyaType; import mvm.rya.api.domain.RyaURI; @@ -322,4 +322,4 @@ private static void loadDataIntoRya(final RepositoryConnection ryaConn, final Se } } } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/pom.xml b/extras/rya.pcj.fluo/pcj.fluo.integration/pom.xml index 6ca8cd7bc..ef667a9e9 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/pom.xml +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/pom.xml @@ -44,7 +44,7 @@ - io.fluo + org.apache.fluo fluo-mini test @@ -54,7 +54,7 @@ test - io.fluo + org.apache.fluo fluo-api diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java index 0b24abcba..9e2b9c66d 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java @@ -63,20 +63,19 @@ import org.openrdf.sail.Sail; import com.google.common.io.Files; - -import io.fluo.api.client.FluoAdmin; -import io.fluo.api.client.FluoAdmin.AlreadyInitializedException; -import io.fluo.api.client.FluoAdmin.TableExistsException; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.client.Snapshot; -import io.fluo.api.config.FluoConfiguration; -import io.fluo.api.config.ObserverConfiguration; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Bytes; -import io.fluo.api.iterator.ColumnIterator; -import io.fluo.api.iterator.RowIterator; -import io.fluo.api.mini.MiniFluo; +import org.apache.fluo.api.client.FluoAdmin; +import org.apache.fluo.api.client.FluoAdmin.AlreadyInitializedException; +import org.apache.fluo.api.client.FluoAdmin.TableExistsException; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.config.ObserverConfiguration; +import org.apache.fluo.api.config.ScannerConfiguration; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.iterator.ColumnIterator; +import org.apache.fluo.api.iterator.RowIterator; +import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.client.Install.InstallConfiguration; import mvm.rya.api.client.RyaClient; @@ -418,4 +417,4 @@ protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExis new FluoAdmin.InitOpts().setClearTable(true).setClearZookeeper(true) ); return FluoFactory.newMiniFluo(config); } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java index b8e068c24..630b86d6d 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java @@ -29,13 +29,13 @@ import com.google.common.base.Optional; -import io.fluo.api.client.FluoAdmin; -import io.fluo.api.client.FluoAdmin.AlreadyInitializedException; -import io.fluo.api.client.FluoAdmin.TableExistsException; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.config.FluoConfiguration; -import io.fluo.api.config.ObserverConfiguration; -import io.fluo.api.mini.MiniFluo; +import org.apache.fluo.api.client.FluoAdmin; +import org.apache.fluo.api.client.FluoAdmin.AlreadyInitializedException; +import org.apache.fluo.api.client.FluoAdmin.TableExistsException; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.config.ObserverConfiguration; +import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.domain.RyaURI; diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java index 8434a8d5b..99e2191c2 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java @@ -32,7 +32,7 @@ import com.beust.jcommander.internal.Lists; -import io.fluo.api.types.TypedTransaction; +import org.apache.fluo.recipes.core.types.TypedTransaction; /** * Integration tests the methods of {@link ListQueryIds}. diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java index 5cb68d360..d5ed4471d 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAOIT.java @@ -30,8 +30,8 @@ import org.openrdf.query.parser.sparql.SPARQLParser; import org.openrdf.repository.RepositoryException; -import io.fluo.api.client.Snapshot; -import io.fluo.api.client.Transaction; +import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.Transaction; /** * Integration tests the methods of {@link FluoQueryMetadataDAO}. diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java index d173e82d3..dd634fbc3 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaExportIT.java @@ -113,4 +113,4 @@ public void resultsExported() throws Exception { // Verify the end results of the query match the expected results. assertEquals(expected, results); } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java index 8cc74040b..bc83eedb9 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java @@ -37,7 +37,7 @@ import com.google.common.collect.Sets; -import io.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoClient; import mvm.rya.accumulo.AccumuloRyaDAO; import mvm.rya.indexing.external.PrecomputedJoinIndexer; @@ -209,4 +209,4 @@ public void historicAndStreamMultiVariables() throws Exception { Set results = getQueryBindingSetValues(fluoClient, sparql); assertEquals(expected, results); } -} \ No newline at end of file +} diff --git a/pom.xml b/pom.xml index 8afa58954..6e280c858 100644 --- a/pom.xml +++ b/pom.xml @@ -119,7 +119,8 @@ under the License. 3.0.4 - 1.0.0-beta-2 + 1.0.0-incubating + 1.0.0-incubating-SNAPSHOT 1.13 3.0.1 @@ -509,20 +510,25 @@ under the License. - io.fluo + org.apache.fluo fluo-api ${fluo.version} - io.fluo + org.apache.fluo fluo-core ${fluo.version} - io.fluo + org.apache.fluo fluo-mini ${fluo.version} + + org.apache.fluo + fluo-recipes-core + ${fluo-recipes.version} + org.mockito From 6a20a6a9bf2140dee5b9496bb2892aa160933bf8 Mon Sep 17 00:00:00 2001 From: Keith Turner Date: Thu, 6 Oct 2016 18:55:34 -0400 Subject: [PATCH 2/4] RYA-142 Stopped using Fluo TypeLayer --- .../rya/indexing/pcj/fluo/api/CreatePcj.java | 22 ++-- .../rya/indexing/pcj/fluo/api/DeletePcj.java | 30 +++--- .../indexing/pcj/fluo/api/GetPcjMetadata.java | 12 +-- .../indexing/pcj/fluo/api/InsertTriples.java | 19 +--- .../indexing/pcj/fluo/api/ListQueryIds.java | 5 +- extras/rya.pcj.fluo/pcj.fluo.app/pom.xml | 4 - .../pcj/fluo/app/FilterResultUpdater.java | 8 +- .../indexing/pcj/fluo/app/IncUpdateDAO.java | 9 +- .../pcj/fluo/app/JoinResultUpdater.java | 10 +- .../pcj/fluo/app/QueryResultUpdater.java | 10 +- .../pcj/fluo/app/StringTypeLayer.java | 29 ----- .../app/export/IncrementalResultExporter.java | 4 +- .../app/export/rya/RyaResultExporter.java | 7 +- .../fluo/app/observers/BindingSetUpdater.java | 12 +-- .../app/observers/QueryResultObserver.java | 18 ++-- .../fluo/app/observers/TripleObserver.java | 30 ++---- .../fluo/app/query/FluoQueryMetadataDAO.java | 100 +++++++++--------- .../indexing/pcj/fluo/api/ListQueryIdsIT.java | 13 ++- pom.xml | 6 -- 19 files changed, 127 insertions(+), 221 deletions(-) delete mode 100644 extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java index 943a0220b..d31e57802 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/CreatePcj.java @@ -29,7 +29,6 @@ import javax.annotation.ParametersAreNonnullByDefault; import org.apache.rya.indexing.pcj.fluo.app.FluoStringConverter; -import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQuery; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO; @@ -55,7 +54,7 @@ import info.aduna.iteration.CloseableIteration; import org.apache.fluo.api.client.FluoClient; -import org.apache.fluo.recipes.core.types.TypedTransaction; +import org.apache.fluo.api.client.Transaction; /** * Sets up a new Pre Computed Join (PCJ) in Fluo from a SPARQL query. @@ -75,11 +74,6 @@ @ParametersAreNonnullByDefault public class CreatePcj { - /** - * Wraps Fluo {@link Transaction}s so that we can write String values to them. - */ - private static final StringTypeLayer STRING_TYPED_LAYER = new StringTypeLayer(); - /** * The default Statement Pattern batch insert size is 1000. */ @@ -150,15 +144,15 @@ public void withRyaIntegration( final ParsedQuery parsedQuery = new SPARQLParser().parseQuery(sparql, null); final FluoQuery fluoQuery = new SparqlFluoQueryBuilder().make(parsedQuery, nodeIds); - try(TypedTransaction tx = STRING_TYPED_LAYER.wrap( fluo.newTransaction() )) { + try(Transaction tx = fluo.newTransaction()) { // Write the query's structure to Fluo. new FluoQueryMetadataDAO().write(tx, fluoQuery); // The results of the query are eventually exported to an instance of Rya, so store the Rya ID for the PCJ. final String queryId = fluoQuery.getQueryMetadata().getNodeId(); - tx.mutate().row(queryId).col(FluoQueryColumns.RYA_PCJ_ID).set(pcjId); - tx.mutate().row(pcjId).col(FluoQueryColumns.PCJ_ID_QUERY_ID).set(queryId); - + tx.set(queryId, FluoQueryColumns.RYA_PCJ_ID, pcjId); + tx.set(pcjId, FluoQueryColumns.PCJ_ID_QUERY_ID, queryId); + // Flush the changes to Fluo. tx.commit(); } @@ -206,7 +200,7 @@ private static void writeBatch(final FluoClient fluo, final StatementPatternMeta final BindingSetStringConverter converter = new BindingSetStringConverter(); - try(TypedTransaction tx = STRING_TYPED_LAYER.wrap(fluo.newTransaction())) { + try(Transaction tx = fluo.newTransaction()) { // Get the node's variable order. final String spNodeId = spMetadata.getNodeId(); final VariableOrder varOrder = spMetadata.getVariableOrder(); @@ -221,9 +215,7 @@ private static void writeBatch(final FluoClient fluo, final StatementPatternMeta final String bindingSetStr = converter.convert(spBindingSet, varOrder); // Write the binding set entry to Fluo for the statement pattern. - tx.mutate().row(spNodeId + NODEID_BS_DELIM + bindingSetStr) - .col(FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET) - .set(bindingSetStr); + tx.set(spNodeId + NODEID_BS_DELIM + bindingSetStr, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET, bindingSetStr); } tx.commit(); diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java index e1a9b8e3b..79ca0ea38 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/DeletePcj.java @@ -28,7 +28,6 @@ import javax.annotation.ParametersAreNonnullByDefault; import org.apache.rya.indexing.pcj.fluo.app.NodeType; -import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.apache.rya.indexing.pcj.fluo.app.query.FilterMetadata; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO; @@ -43,7 +42,6 @@ import org.apache.fluo.api.data.Column; import org.apache.fluo.api.data.RowColumnValue; import org.apache.fluo.api.data.Span; -import org.apache.fluo.recipes.core.types.TypedTransaction; /** * Deletes a Pre-computed Join (PCJ) from Fluo. @@ -174,7 +172,7 @@ private void deleteMetadata(final Transaction tx, final List nodeIds, fi requireNonNull(nodeIds); requireNonNull(pcjId); - try (final TypedTransaction typeTx = new StringTypeLayer().wrap(tx)) { + try (final Transaction typeTx = tx) { deletePcjIdAndSparqlMetadata(typeTx, pcjId); for (final String nodeId : nodeIds) { @@ -192,7 +190,7 @@ private void deleteMetadata(final Transaction tx, final List nodeIds, fi * @param nodeId - The Node ID of the query node to delete. (not null) * @param columns - The columns that will be deleted. (not null) */ - private void deleteMetadataColumns(final TypedTransaction tx, final String nodeId, final List columns) { + private void deleteMetadataColumns(final Transaction tx, final String nodeId, final List columns) { requireNonNull(tx); requireNonNull(columns); requireNonNull(nodeId); @@ -211,15 +209,15 @@ private void deleteMetadataColumns(final TypedTransaction tx, final String nodeI * @param tx - Transaction the deletes will be performed with. (not null) * @param pcjId - The PCJ whose metadata will be deleted. (not null) */ - private void deletePcjIdAndSparqlMetadata(final TypedTransaction tx, final String pcjId) { + private void deletePcjIdAndSparqlMetadata(final Transaction tx, final String pcjId) { requireNonNull(tx); requireNonNull(pcjId); final String queryId = getQueryIdFromPcjId(tx, pcjId); final String sparql = getSparqlFromQueryId(tx, queryId); - tx.delete(Bytes.of(queryId), FluoQueryColumns.RYA_PCJ_ID); - tx.delete(Bytes.of(sparql), FluoQueryColumns.QUERY_ID); - tx.delete(Bytes.of(pcjId), FluoQueryColumns.PCJ_ID_QUERY_ID); + tx.delete(queryId, FluoQueryColumns.RYA_PCJ_ID); + tx.delete(sparql, FluoQueryColumns.QUERY_ID); + tx.delete(pcjId, FluoQueryColumns.PCJ_ID_QUERY_ID); } @@ -253,17 +251,19 @@ private boolean deleteDataBatch(final Transaction tx, final CellScanner scanner, requireNonNull(scanner); requireNonNull(column); - int count = 0; - Iterator iter = scanner.iterator(); - while (iter.hasNext() && count < batchSize) { + try(Transaction ntx = tx) { + int count = 0; + Iterator iter = scanner.iterator(); + while (iter.hasNext() && count < batchSize) { final Bytes row = iter.next().getRow(); count++; tx.delete(row, column); - } + } - final boolean hasNext = iter.hasNext(); - tx.commit(); - return hasNext; + final boolean hasNext = iter.hasNext(); + tx.commit(); + return hasNext; + } } private String getQueryIdFromPcjId(final Transaction tx, final String pcjId) { diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java index d8c800e66..061a1d54c 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/GetPcjMetadata.java @@ -24,15 +24,14 @@ import java.util.HashMap; import java.util.Map; -import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; import org.apache.rya.indexing.pcj.storage.PcjException; import org.apache.rya.indexing.pcj.storage.PcjMetadata; import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage; import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.Snapshot; import org.apache.fluo.api.data.Bytes; -import org.apache.fluo.recipes.core.types.TypedSnapshot; /** * Get {@link PcjMetadata} for queries that are managed by the Fluo app. @@ -87,13 +86,12 @@ public PcjMetadata getMetadata(final PrecomputedJoinStorage pcjStorage, final Fl // Lookup the Rya PCJ ID associated with the query. String pcjId = null; - try(TypedSnapshot snap = new StringTypeLayer().wrap( fluo.newSnapshot() ) ) { - final Bytes pcjIdBytes = snap.get(Bytes.of(queryId), FluoQueryColumns.RYA_PCJ_ID); - if(pcjIdBytes == null) { + try(Snapshot snap = fluo.newSnapshot() ) { + pcjId = snap.gets(queryId, FluoQueryColumns.RYA_PCJ_ID); + if(pcjId == null) { throw new NotInFluoException("Could not get the PcjMetadata for queryId '" + queryId + "' because a Rya PCJ ID not stored in the Fluo table."); } - pcjId = pcjIdBytes.toString(); } // Fetch the metadata from the storage. @@ -128,4 +126,4 @@ public NotInAccumuloException(final String message, final Exception cause) { super(message, cause); } } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java index 1f4538829..b184ff35d 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/InsertTriples.java @@ -25,15 +25,13 @@ import java.util.Map; import org.apache.log4j.Logger; -import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; import com.google.common.base.Optional; import org.apache.fluo.api.client.FluoClient; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; -import org.apache.fluo.recipes.core.types.TypedTransaction; +import org.apache.fluo.api.client.Transaction; +import org.apache.fluo.api.data.Bytes; import mvm.rya.api.RdfCloudTripleStoreConstants.TABLE_LAYOUT; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.resolver.triple.TripleRow; @@ -47,18 +45,11 @@ public class InsertTriples { private static final Logger log = Logger.getLogger(InsertTriples.class); - /** - * Wraps Fluo {@link Transaction}s so that we can write String values to them. - */ - private static final StringTypeLayer STRING_TYPED_LAYER = new StringTypeLayer(); - /** * Converts triples into the byte[] used as the row ID in Accumulo. */ private static final WholeRowTripleResolver TRIPLE_RESOLVER = new WholeRowTripleResolver(); - private static final Encoder ENCODER = new StringEncoder(); - // TODO visiblity is part of RyaStatement. Put it there instead. /** @@ -85,10 +76,10 @@ public void insert(final FluoClient fluo, final Collection triples checkNotNull(triples); checkNotNull(visibility); - try(TypedTransaction tx = STRING_TYPED_LAYER.wrap(fluo.newTransaction())) { + try(Transaction tx = fluo.newTransaction()) { for(final RyaStatement triple : triples) { try { - tx.mutate().row(spoFormat(triple)).col(FluoQueryColumns.TRIPLES).set(ENCODER.encode(visibility.or(""))); + tx.set(Bytes.of(spoFormat(triple)), FluoQueryColumns.TRIPLES, Bytes.of(visibility.or(""))); } catch (final TripleRowResolverException e) { log.error("Could not convert a Triple into the SPO format: " + triple); } @@ -111,4 +102,4 @@ public static byte[] spoFormat(final RyaStatement triple) throws TripleRowResolv final TripleRow spoRow = serialized.get(TABLE_LAYOUT.SPO); return spoRow.getRow(); } -} \ No newline at end of file +} diff --git a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java index 3913e410a..df1648b51 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java +++ b/extras/rya.pcj.fluo/pcj.fluo.api/src/main/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIds.java @@ -24,13 +24,12 @@ import java.util.Collections; import java.util.List; -import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.Snapshot; import org.apache.fluo.api.client.scanner.CellScanner; import org.apache.fluo.api.data.RowColumnValue; -import org.apache.fluo.recipes.core.types.TypedSnapshot; /** * Finds all queries that are being managed by this instance of Fluo that @@ -51,7 +50,7 @@ public List listQueryIds(final FluoClient fluo) { final List queryIds = new ArrayList<>(); - try(TypedSnapshot snap = new StringTypeLayer().wrap( fluo.newSnapshot() )) { + try(Snapshot snap = fluo.newSnapshot() ) { // Create an iterator that iterates over the QUERY_ID column. final CellScanner cellScanner = snap.scanner().fetch( FluoQueryColumns.QUERY_ID).build(); diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml b/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml index 45ea9ce28..f756cdbfb 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml +++ b/extras/rya.pcj.fluo/pcj.fluo.app/pom.xml @@ -55,10 +55,6 @@ under the License. org.apache.fluo fluo-api - - org.apache.fluo - fluo-recipes-core - org.apache.fluo fluo-core diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java index 328c6537c..b1af4fc2c 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/FilterResultUpdater.java @@ -52,8 +52,6 @@ import org.apache.fluo.api.client.TransactionBase; import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; /** * Updates the results of a Filter node when its child has added a new Binding @@ -62,8 +60,6 @@ @ParametersAreNonnullByDefault public class FilterResultUpdater { - private static final Encoder ENCODER = new StringEncoder(); - private static final BindingSetStringConverter ID_CONVERTER = new BindingSetStringConverter(); private static final VisibilityBindingSetStringConverter VALUE_CONVERTER = new VisibilityBindingSetStringConverter(); @@ -134,9 +130,9 @@ public void updateFilterResults( String filterBindingSetValueString = ""; filterBindingSetValueString = VALUE_CONVERTER.convert(childBindingSet, filterVarOrder); - final Bytes row = ENCODER.encode( filterMetadata.getNodeId() + NODEID_BS_DELIM + filterBindingSetIdString ); + final String row = filterMetadata.getNodeId() + NODEID_BS_DELIM + filterBindingSetIdString; final Column col = FluoQueryColumns.FILTER_BINDING_SET; - final Bytes value = ENCODER.encode(filterBindingSetValueString); + final String value = filterBindingSetValueString; tx.set(row, col, value); } } diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java index 7ccfeff6e..513ab40c9 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/IncUpdateDAO.java @@ -25,11 +25,11 @@ import org.apache.fluo.api.client.FluoClient; import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.Transaction; import org.apache.fluo.api.client.scanner.CellScanner; import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; import org.apache.fluo.api.data.RowColumnValue; -import org.apache.fluo.recipes.core.types.TypedTransaction; import mvm.rya.api.RdfCloudTripleStoreConstants.TABLE_LAYOUT; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.resolver.triple.TripleRow; @@ -38,7 +38,6 @@ public class IncUpdateDAO { - private static final StringTypeLayer stl = new StringTypeLayer(); private static final WholeRowTripleResolver tr = new WholeRowTripleResolver(); public static RyaStatement deserializeTriple(final Bytes row) { @@ -77,7 +76,7 @@ public static String getTripleString(final Bytes row) { */ public static void addRow(final FluoClient fluoClient, final String row, final Column col, final String val) { checkNotNull(fluoClient); - try (TypedTransaction tx = stl.wrap(fluoClient.newTransaction())) { + try (Transaction tx = fluoClient.newTransaction()) { addRow(tx, row, col, val); tx.commit(); } @@ -91,9 +90,9 @@ public static void addRow(final FluoClient fluoClient, final String row, final C * @param col - The Column. * @param val - The value. */ - public static void addRow(final TypedTransaction tx, final String row, final Column col, final String val) { + public static void addRow(final Transaction tx, final String row, final Column col, final String val) { checkNotNull(tx); - tx.mutate().row(row).col(col).set(val); + tx.set(row, col, val); } /** diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java index 5ac69b013..73a03ca57 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/JoinResultUpdater.java @@ -49,12 +49,9 @@ import org.apache.fluo.api.client.TransactionBase; import org.apache.fluo.api.client.scanner.ColumnScanner; import org.apache.fluo.api.client.scanner.RowScanner; -import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; import org.apache.fluo.api.data.ColumnValue; import org.apache.fluo.api.data.Span; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; /** * Updates the results of a Join node when one of its children has added a @@ -67,8 +64,7 @@ public class JoinResultUpdater { private static final VisibilityBindingSetStringConverter valueConverter = new VisibilityBindingSetStringConverter(); private final FluoQueryMetadataDAO queryDao = new FluoQueryMetadataDAO(); - private final Encoder encoder = new StringEncoder(); - + /** * Updates the results of a Join node when one of its children has added a * new Binding Set to its results. @@ -132,9 +128,9 @@ public void updateJoinResults( final String joinBindingSetStringId = idConverter.convert(newJoinResult, joinVarOrder); final String joinBindingSetStringValue = valueConverter.convert(newJoinResult, joinVarOrder); - final Bytes row = encoder.encode(joinMetadata.getNodeId() + NODEID_BS_DELIM + joinBindingSetStringId); + final String row = joinMetadata.getNodeId() + NODEID_BS_DELIM + joinBindingSetStringId; final Column col = FluoQueryColumns.JOIN_BINDING_SET; - final Bytes value = encoder.encode(joinBindingSetStringValue); + final String value = joinBindingSetStringValue; tx.set(row, col, value); } } diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java index 41f902598..b4800fceb 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/QueryResultUpdater.java @@ -33,10 +33,7 @@ import org.openrdf.query.impl.MapBindingSet; import org.apache.fluo.api.client.TransactionBase; -import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; /** * Updates the results of a Query node when one of its children has added a @@ -44,8 +41,7 @@ */ @ParametersAreNonnullByDefault public class QueryResultUpdater { - private final Encoder encoder = new StringEncoder(); - + private final BindingSetStringConverter converter = new BindingSetStringConverter(); private final VisibilityBindingSetStringConverter valueConverter = new VisibilityBindingSetStringConverter(); @@ -79,9 +75,9 @@ public void updateQueryResults( final String queryBindingSetValueString = valueConverter.convert(new VisibilityBindingSet(queryBindingSet, childBindingSet.getVisibility()), queryVarOrder); // Commit it to the Fluo table for the SPARQL query. This isn't guaranteed to be a new entry. - final Bytes row = encoder.encode(queryMetadata.getNodeId() + NODEID_BS_DELIM + queryBindingSetString); + final String row = queryMetadata.getNodeId() + NODEID_BS_DELIM + queryBindingSetString; final Column col = FluoQueryColumns.QUERY_BINDING_SET; - final Bytes value = encoder.encode(queryBindingSetValueString); + final String value = queryBindingSetValueString; tx.set(row, col, value); } } \ No newline at end of file diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java deleted file mode 100644 index aecb434c4..000000000 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/StringTypeLayer.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.indexing.pcj.fluo.app; - -import org.apache.fluo.recipes.core.types.StringEncoder; -import org.apache.fluo.recipes.core.types.TypeLayer; - -public class StringTypeLayer extends TypeLayer { - - public StringTypeLayer() { - super(new StringEncoder()); - } -} \ No newline at end of file diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java index a7f016d8e..ecc39bdda 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/IncrementalResultExporter.java @@ -20,9 +20,9 @@ import javax.annotation.ParametersAreNonnullByDefault; +import org.apache.fluo.api.client.TransactionBase; import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet; -import org.apache.fluo.recipes.core.types.TypedTransactionBase; /** * Exports a single Binding Set that is a new result for a SPARQL query to some @@ -40,7 +40,7 @@ public interface IncrementalResultExporter { * Fluo application. (not null) * @throws ResultExportException The result could not be exported. */ - public void export(TypedTransactionBase tx, String queryId, VisibilityBindingSet result) throws ResultExportException; + public void export(TransactionBase tx, String queryId, VisibilityBindingSet result) throws ResultExportException; /** * A result could not be exported. diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java index 27530f006..a4b589f0a 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/export/rya/RyaResultExporter.java @@ -27,9 +27,8 @@ import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage; import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage.PCJStorageException; import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSet; - +import org.apache.fluo.api.client.TransactionBase; import org.apache.fluo.api.data.Bytes; -import org.apache.fluo.recipes.core.types.TypedTransactionBase; /** * Incrementally exports SPARQL query results to Accumulo PCJ tables as they are defined by Rya. @@ -49,14 +48,14 @@ public RyaResultExporter(final PrecomputedJoinStorage pcjStorage) { @Override public void export( - final TypedTransactionBase fluoTx, + final TransactionBase fluoTx, final String queryId, final VisibilityBindingSet result) throws ResultExportException { checkNotNull(fluoTx); checkNotNull(queryId); checkNotNull(result); - final String pcjId = fluoTx.get(Bytes.of(queryId), FluoQueryColumns.RYA_PCJ_ID).toString(); + final String pcjId = fluoTx.gets(queryId, FluoQueryColumns.RYA_PCJ_ID); try { pcjStorage.addResults(pcjId, Collections.singleton(result)); diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java index e344b0ab9..a2953cf7b 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/BindingSetUpdater.java @@ -37,10 +37,7 @@ import org.apache.fluo.api.client.TransactionBase; import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; -import org.apache.fluo.recipes.core.types.TypedObserver; -import org.apache.fluo.recipes.core.types.TypedTransactionBase; +import org.apache.fluo.api.observer.AbstractObserver; /** * Notified when the results of a node have been updated to include a new Binding @@ -48,9 +45,8 @@ * results. */ @ParametersAreNonnullByDefault -public abstract class BindingSetUpdater extends TypedObserver { +public abstract class BindingSetUpdater extends AbstractObserver { - private final Encoder encoder = new StringEncoder(); // DAO private final FluoQueryMetadataDAO queryDao = new FluoQueryMetadataDAO(); @@ -72,12 +68,12 @@ public abstract class BindingSetUpdater extends TypedObserver { public abstract Observation parseObservation(TransactionBase tx, final BindingSetRow parsedRow); @Override - public final void process(final TypedTransactionBase tx, final Bytes row, final Column col) { + public final void process(final TransactionBase tx, final Bytes row, final Column col) { checkNotNull(tx); checkNotNull(row); checkNotNull(col); - final String bindingSetString = encoder.decodeString(tx.get(row, col)); + final String bindingSetString = tx.get(row, col).toString(); final Observation observation = parseObservation( tx, new BindingSetRow(BindingSetRow.make(row).getNodeId(), bindingSetString) ); final String observedNodeId = observation.getObservedNodeId(); final VisibilityBindingSet observedBindingSet = observation.getObservedBindingSet(); diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java index 638b1fcc9..0944f9b87 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/QueryResultObserver.java @@ -38,23 +38,19 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableSet; - +import org.apache.fluo.api.client.TransactionBase; import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; -import org.apache.fluo.recipes.core.types.TypedObserver; -import org.apache.fluo.recipes.core.types.TypedTransactionBase; +import org.apache.fluo.api.observer.AbstractObserver; import mvm.rya.accumulo.utils.VisibilitySimplifier; /** * Performs incremental result exporting to the configured destinations. */ -public class QueryResultObserver extends TypedObserver { +public class QueryResultObserver extends AbstractObserver { private static final Logger log = Logger.getLogger(QueryResultObserver.class); private static final FluoQueryMetadataDAO QUERY_DAO = new FluoQueryMetadataDAO(); - private static final Encoder ENCODER = new StringEncoder(); private static final VisibilityBindingSetStringConverter CONVERTER = new VisibilityBindingSetStringConverter(); /** @@ -107,11 +103,13 @@ public void init(final Context context) { } @Override - public void process(final TypedTransactionBase tx, final Bytes row, final Column col) { + public void process(final TransactionBase tx, final Bytes brow, final Column col) { + final String row = brow.toString(); + // Read the SPARQL query and it Binding Set from the row id. - final String[] queryAndBindingSet = ENCODER.decodeString(row).split(NODEID_BS_DELIM); + final String[] queryAndBindingSet = row.split(NODEID_BS_DELIM); final String queryId = queryAndBindingSet[0]; - final String bindingSetString = ENCODER.decodeString(tx.get(row, col)); + final String bindingSetString = tx.gets(row, col); // Fetch the query's Variable Order from the Fluo table. final QueryMetadata queryMetadata = QUERY_DAO.readQueryMetadata(tx, queryId); diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java index 31a4c29e1..70f1cbd18 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/observers/TripleObserver.java @@ -26,7 +26,6 @@ import java.util.Map; import org.apache.rya.indexing.pcj.fluo.app.IncUpdateDAO; -import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryColumns; import org.apache.rya.indexing.pcj.fluo.app.query.FluoQueryMetadataDAO; import org.apache.rya.indexing.pcj.fluo.app.query.StatementPatternMetadata; @@ -35,32 +34,26 @@ import org.apache.rya.indexing.pcj.storage.accumulo.VisibilityBindingSetStringConverter; import com.google.common.collect.Maps; - +import org.apache.fluo.api.client.TransactionBase; import org.apache.fluo.api.client.scanner.ColumnScanner; import org.apache.fluo.api.client.scanner.RowScanner; import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; import org.apache.fluo.api.data.ColumnValue; import org.apache.fluo.api.data.Span; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; -import org.apache.fluo.recipes.core.types.TypedObserver; -import org.apache.fluo.recipes.core.types.TypedTransactionBase; +import org.apache.fluo.api.observer.AbstractObserver; /** * An observer that matches new Triples to the Statement Patterns that are part * of any PCJ that is being maintained. If the triple matches a pattern, then * the new result is stored as a binding set for the pattern. */ -public class TripleObserver extends TypedObserver { +public class TripleObserver extends AbstractObserver { - private static final Encoder ENCODER = new StringEncoder(); private static final FluoQueryMetadataDAO QUERY_DAO = new FluoQueryMetadataDAO(); private static final VisibilityBindingSetStringConverter CONVERTER = new VisibilityBindingSetStringConverter(); - public TripleObserver() { - super(new StringTypeLayer()); - } + public TripleObserver() {} @Override public ObservedColumn getObservedColumn() { @@ -68,15 +61,12 @@ public ObservedColumn getObservedColumn() { } @Override - public void process(final TypedTransactionBase tx, final Bytes row, final Column column) { + public void process(final TransactionBase tx, final Bytes brow, final Column column) { //get string representation of triple - final String triple = IncUpdateDAO.getTripleString(row); - final Bytes visiBytes = tx.get(row, FluoQueryColumns.TRIPLES); - String visibility = ""; - if(visiBytes != null) { - visibility = ENCODER.decodeString(visiBytes); - } - + String row = brow.toString(); + final String triple = IncUpdateDAO.getTripleString(brow); + String visibility = tx.gets(row, FluoQueryColumns.TRIPLES, ""); + //get variable metadata for all SP in table RowScanner rscanner = tx.scanner().over(Span.prefix(SP_PREFIX)).fetch(FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER).byRow().build(); @@ -100,7 +90,7 @@ public void process(final TypedTransactionBase tx, final Bytes row, final Column CONVERTER.convert(bindingSetString, varOrder), visibility); final String valueString = CONVERTER.convert(bindingSet, varOrder); - tx.mutate().row(spID + NODEID_BS_DELIM + bindingSetString).col(FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET).set(valueString); + tx.set(spID + NODEID_BS_DELIM + bindingSetString, FluoQueryColumns.STATEMENT_PATTERN_BINDING_SET, valueString); } } } diff --git a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java index a955a538c..8d41c6124 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java +++ b/extras/rya.pcj.fluo/pcj.fluo.app/src/main/java/org/apache/rya/indexing/pcj/fluo/app/query/FluoQueryMetadataDAO.java @@ -34,8 +34,6 @@ import org.apache.fluo.api.client.TransactionBase; import org.apache.fluo.api.data.Bytes; import org.apache.fluo.api.data.Column; -import org.apache.fluo.recipes.core.types.Encoder; -import org.apache.fluo.recipes.core.types.StringEncoder; /** * Reads and writes {@link FluoQuery} instances and their components to/from @@ -44,8 +42,6 @@ @ParametersAreNonnullByDefault public class FluoQueryMetadataDAO { - private static final Encoder encoder = new StringEncoder(); - /** * Write an instance of {@link QueryMetadata} to the Fluo table. * @@ -56,11 +52,11 @@ public void write(final TransactionBase tx, final QueryMetadata metadata) { checkNotNull(tx); checkNotNull(metadata); - final Bytes rowId = encoder.encode(metadata.getNodeId()); + final String rowId = metadata.getNodeId(); tx.set(rowId, FluoQueryColumns.QUERY_NODE_ID, rowId); - tx.set(rowId, FluoQueryColumns.QUERY_VARIABLE_ORDER, encoder.encode( metadata.getVariableOrder().toString() )); - tx.set(rowId, FluoQueryColumns.QUERY_SPARQL, encoder.encode( metadata.getSparql() )); - tx.set(rowId, FluoQueryColumns.QUERY_CHILD_NODE_ID, encoder.encode( metadata.getChildNodeId() )); + tx.set(rowId, FluoQueryColumns.QUERY_VARIABLE_ORDER, metadata.getVariableOrder().toString()); + tx.set(rowId, FluoQueryColumns.QUERY_SPARQL, metadata.getSparql() ); + tx.set(rowId, FluoQueryColumns.QUERY_CHILD_NODE_ID, metadata.getChildNodeId() ); } /** @@ -79,18 +75,18 @@ private QueryMetadata.Builder readQueryMetadataBuilder(final SnapshotBase sx, fi checkNotNull(nodeId); // Fetch the values from the Fluo table. - final Bytes rowId = encoder.encode(nodeId); - final Map values = sx.get(rowId, Sets.newHashSet( + final String rowId = nodeId; + final Map values = sx.gets(rowId, FluoQueryColumns.QUERY_VARIABLE_ORDER, FluoQueryColumns.QUERY_SPARQL, - FluoQueryColumns.QUERY_CHILD_NODE_ID)); + FluoQueryColumns.QUERY_CHILD_NODE_ID); // Return an object holding them. - final String varOrderString = encoder.decodeString( values.get(FluoQueryColumns.QUERY_VARIABLE_ORDER)); + final String varOrderString = values.get(FluoQueryColumns.QUERY_VARIABLE_ORDER); final VariableOrder varOrder = new VariableOrder(varOrderString); - final String sparql = encoder.decodeString( values.get(FluoQueryColumns.QUERY_SPARQL) ); - final String childNodeId = encoder.decodeString( values.get(FluoQueryColumns.QUERY_CHILD_NODE_ID) ); + final String sparql = values.get(FluoQueryColumns.QUERY_SPARQL); + final String childNodeId = values.get(FluoQueryColumns.QUERY_CHILD_NODE_ID); return QueryMetadata.builder(nodeId) .setVariableOrder( varOrder ) @@ -108,13 +104,13 @@ public void write(final TransactionBase tx, final FilterMetadata metadata) { checkNotNull(tx); checkNotNull(metadata); - final Bytes rowId = encoder.encode(metadata.getNodeId()); + final String rowId = metadata.getNodeId(); tx.set(rowId, FluoQueryColumns.FILTER_NODE_ID, rowId); - tx.set(rowId, FluoQueryColumns.FILTER_VARIABLE_ORDER, encoder.encode( metadata.getVariableOrder().toString() )); - tx.set(rowId, FluoQueryColumns.FILTER_ORIGINAL_SPARQL, encoder.encode( metadata.getOriginalSparql() )); - tx.set(rowId, FluoQueryColumns.FILTER_INDEX_WITHIN_SPARQL, encoder.encode( metadata.getFilterIndexWithinSparql() )); - tx.set(rowId, FluoQueryColumns.FILTER_PARENT_NODE_ID, encoder.encode( metadata.getParentNodeId() )); - tx.set(rowId, FluoQueryColumns.FILTER_CHILD_NODE_ID, encoder.encode( metadata.getChildNodeId() )); + tx.set(rowId, FluoQueryColumns.FILTER_VARIABLE_ORDER, metadata.getVariableOrder().toString()); + tx.set(rowId, FluoQueryColumns.FILTER_ORIGINAL_SPARQL, metadata.getOriginalSparql() ); + tx.set(rowId, FluoQueryColumns.FILTER_INDEX_WITHIN_SPARQL, metadata.getFilterIndexWithinSparql()+"" ); + tx.set(rowId, FluoQueryColumns.FILTER_PARENT_NODE_ID, metadata.getParentNodeId() ); + tx.set(rowId, FluoQueryColumns.FILTER_CHILD_NODE_ID, metadata.getChildNodeId() ); } /** @@ -133,22 +129,22 @@ private FilterMetadata.Builder readFilterMetadataBuilder(final SnapshotBase sx, checkNotNull(nodeId); // Fetch the values from the Fluo table. - final Bytes rowId = encoder.encode(nodeId); - final Map values = sx.get(rowId, Sets.newHashSet( + final String rowId = nodeId; + final Map values = sx.gets(rowId, FluoQueryColumns.FILTER_VARIABLE_ORDER, FluoQueryColumns.FILTER_ORIGINAL_SPARQL, FluoQueryColumns.FILTER_INDEX_WITHIN_SPARQL, FluoQueryColumns.FILTER_PARENT_NODE_ID, - FluoQueryColumns.FILTER_CHILD_NODE_ID)); + FluoQueryColumns.FILTER_CHILD_NODE_ID); // Return an object holding them. - final String varOrderString = encoder.decodeString( values.get(FluoQueryColumns.FILTER_VARIABLE_ORDER)); + final String varOrderString = values.get(FluoQueryColumns.FILTER_VARIABLE_ORDER); final VariableOrder varOrder = new VariableOrder(varOrderString); - final String originalSparql = encoder.decodeString( values.get(FluoQueryColumns.FILTER_ORIGINAL_SPARQL) ); - final int filterIndexWithinSparql = encoder.decodeInteger( values.get(FluoQueryColumns.FILTER_INDEX_WITHIN_SPARQL) ); - final String parentNodeId = encoder.decodeString( values.get(FluoQueryColumns.FILTER_PARENT_NODE_ID) ); - final String childNodeId = encoder.decodeString( values.get(FluoQueryColumns.FILTER_CHILD_NODE_ID) ); + final String originalSparql = values.get(FluoQueryColumns.FILTER_ORIGINAL_SPARQL); + final int filterIndexWithinSparql = Integer.parseInt(values.get(FluoQueryColumns.FILTER_INDEX_WITHIN_SPARQL)); + final String parentNodeId = values.get(FluoQueryColumns.FILTER_PARENT_NODE_ID); + final String childNodeId = values.get(FluoQueryColumns.FILTER_CHILD_NODE_ID); return FilterMetadata.builder(nodeId) .setVarOrder(varOrder) @@ -168,13 +164,13 @@ public void write(final TransactionBase tx, final JoinMetadata metadata) { checkNotNull(tx); checkNotNull(metadata); - final Bytes rowId = encoder.encode(metadata.getNodeId()); + final String rowId = metadata.getNodeId(); tx.set(rowId, FluoQueryColumns.JOIN_NODE_ID, rowId); - tx.set(rowId, FluoQueryColumns.JOIN_VARIABLE_ORDER, encoder.encode( metadata.getVariableOrder().toString() )); - tx.set(rowId, FluoQueryColumns.JOIN_TYPE, encoder.encode(metadata.getJoinType().toString()) ); - tx.set(rowId, FluoQueryColumns.JOIN_PARENT_NODE_ID, encoder.encode( metadata.getParentNodeId() )); - tx.set(rowId, FluoQueryColumns.JOIN_LEFT_CHILD_NODE_ID, encoder.encode( metadata.getLeftChildNodeId() )); - tx.set(rowId, FluoQueryColumns.JOIN_RIGHT_CHILD_NODE_ID, encoder.encode( metadata.getRightChildNodeId() )); + tx.set(rowId, FluoQueryColumns.JOIN_VARIABLE_ORDER, metadata.getVariableOrder().toString()); + tx.set(rowId, FluoQueryColumns.JOIN_TYPE, metadata.getJoinType().toString() ); + tx.set(rowId, FluoQueryColumns.JOIN_PARENT_NODE_ID, metadata.getParentNodeId() ); + tx.set(rowId, FluoQueryColumns.JOIN_LEFT_CHILD_NODE_ID, metadata.getLeftChildNodeId() ); + tx.set(rowId, FluoQueryColumns.JOIN_RIGHT_CHILD_NODE_ID, metadata.getRightChildNodeId() ); } /** @@ -193,24 +189,24 @@ private JoinMetadata.Builder readJoinMetadataBuilder(final SnapshotBase sx, fina checkNotNull(nodeId); // Fetch the values from the Fluo table. - final Bytes rowId = encoder.encode(nodeId); - final Map values = sx.get(rowId, Sets.newHashSet( + final String rowId = nodeId; + final Map values = sx.gets(rowId, FluoQueryColumns.JOIN_VARIABLE_ORDER, FluoQueryColumns.JOIN_TYPE, FluoQueryColumns.JOIN_PARENT_NODE_ID, FluoQueryColumns.JOIN_LEFT_CHILD_NODE_ID, - FluoQueryColumns.JOIN_RIGHT_CHILD_NODE_ID)); + FluoQueryColumns.JOIN_RIGHT_CHILD_NODE_ID); // Return an object holding them. - final String varOrderString = encoder.decodeString( values.get(FluoQueryColumns.JOIN_VARIABLE_ORDER)); + final String varOrderString = values.get(FluoQueryColumns.JOIN_VARIABLE_ORDER); final VariableOrder varOrder = new VariableOrder(varOrderString); - final String joinTypeString = encoder.decodeString( values.get(FluoQueryColumns.JOIN_TYPE) ); + final String joinTypeString = values.get(FluoQueryColumns.JOIN_TYPE); final JoinType joinType = JoinType.valueOf(joinTypeString); - final String parentNodeId = encoder.decodeString( values.get(FluoQueryColumns.JOIN_PARENT_NODE_ID) ); - final String leftChildNodeId = encoder.decodeString( values.get(FluoQueryColumns.JOIN_LEFT_CHILD_NODE_ID) ); - final String rightChildNodeId = encoder.decodeString( values.get(FluoQueryColumns.JOIN_RIGHT_CHILD_NODE_ID) ); + final String parentNodeId = values.get(FluoQueryColumns.JOIN_PARENT_NODE_ID); + final String leftChildNodeId = values.get(FluoQueryColumns.JOIN_LEFT_CHILD_NODE_ID); + final String rightChildNodeId = values.get(FluoQueryColumns.JOIN_RIGHT_CHILD_NODE_ID); return JoinMetadata.builder(nodeId) .setVariableOrder(varOrder) @@ -230,11 +226,11 @@ public void write(final TransactionBase tx, final StatementPatternMetadata metad checkNotNull(tx); checkNotNull(metadata); - final Bytes rowId = encoder.encode(metadata.getNodeId()); + final String rowId = metadata.getNodeId(); tx.set(rowId, FluoQueryColumns.STATEMENT_PATTERN_NODE_ID, rowId); - tx.set(rowId, FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER, encoder.encode( metadata.getVariableOrder().toString() )); - tx.set(rowId, FluoQueryColumns.STATEMENT_PATTERN_PATTERN, encoder.encode( metadata.getStatementPattern() )); - tx.set(rowId, FluoQueryColumns.STATEMENT_PATTERN_PARENT_NODE_ID, encoder.encode( metadata.getParentNodeId() )); + tx.set(rowId, FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER, metadata.getVariableOrder().toString()); + tx.set(rowId, FluoQueryColumns.STATEMENT_PATTERN_PATTERN, metadata.getStatementPattern() ); + tx.set(rowId, FluoQueryColumns.STATEMENT_PATTERN_PARENT_NODE_ID, metadata.getParentNodeId()); } /** @@ -253,18 +249,18 @@ private StatementPatternMetadata.Builder readStatementPatternMetadataBuilder(fin checkNotNull(nodeId); // Fetch the values from the Fluo table. - final Bytes rowId = encoder.encode(nodeId); - final Map values = sx.get(rowId, Sets.newHashSet( + final String rowId = nodeId; + final Map values = sx.gets(rowId, FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER, FluoQueryColumns.STATEMENT_PATTERN_PATTERN, - FluoQueryColumns.STATEMENT_PATTERN_PARENT_NODE_ID)); + FluoQueryColumns.STATEMENT_PATTERN_PARENT_NODE_ID); // Return an object holding them. - final String varOrderString = encoder.decodeString( values.get(FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER)); + final String varOrderString = values.get(FluoQueryColumns.STATEMENT_PATTERN_VARIABLE_ORDER); final VariableOrder varOrder = new VariableOrder(varOrderString); - final String pattern = encoder.decodeString( values.get(FluoQueryColumns.STATEMENT_PATTERN_PATTERN) ); - final String parentNodeId = encoder.decodeString( values.get(FluoQueryColumns.STATEMENT_PATTERN_PARENT_NODE_ID) ); + final String pattern = values.get(FluoQueryColumns.STATEMENT_PATTERN_PATTERN); + final String parentNodeId = values.get(FluoQueryColumns.STATEMENT_PATTERN_PARENT_NODE_ID); return StatementPatternMetadata.builder(nodeId) .setVarOrder(varOrder) diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java index 99e2191c2..19bc27284 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/ListQueryIdsIT.java @@ -27,12 +27,11 @@ import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.accumulo.core.client.TableExistsException; import org.apache.rya.indexing.pcj.fluo.ITBase; -import org.apache.rya.indexing.pcj.fluo.app.StringTypeLayer; import org.junit.Test; import com.beust.jcommander.internal.Lists; -import org.apache.fluo.recipes.core.types.TypedTransaction; +import org.apache.fluo.api.client.Transaction; /** * Integration tests the methods of {@link ListQueryIds}. @@ -47,11 +46,11 @@ public class ListQueryIdsIT extends ITBase { @Test public void getQueryIds() throws AccumuloException, AccumuloSecurityException, TableExistsException { // Store a few SPARQL/Query ID pairs in the Fluo table. - try(TypedTransaction tx = new StringTypeLayer().wrap( fluoClient.newTransaction() )) { - tx.mutate().row("SPARQL_3").col(QUERY_ID).set("ID_3"); - tx.mutate().row("SPARQL_1").col(QUERY_ID).set("ID_1"); - tx.mutate().row("SPARQL_4").col(QUERY_ID).set("ID_4"); - tx.mutate().row("SPARQL_2").col(QUERY_ID).set("ID_2"); + try(Transaction tx = fluoClient.newTransaction()) { + tx.set("SPARQL_3", QUERY_ID, "ID_3"); + tx.set("SPARQL_1", QUERY_ID, "ID_1"); + tx.set("SPARQL_4", QUERY_ID, "ID_4"); + tx.set("SPARQL_2", QUERY_ID, "ID_2"); tx.commit(); } diff --git a/pom.xml b/pom.xml index 6e280c858..6ca4505e2 100644 --- a/pom.xml +++ b/pom.xml @@ -120,7 +120,6 @@ under the License. 3.0.4 1.0.0-incubating - 1.0.0-incubating-SNAPSHOT 1.13 3.0.1 @@ -524,11 +523,6 @@ under the License. fluo-mini ${fluo.version} - - org.apache.fluo - fluo-recipes-core - ${fluo-recipes.version} - org.mockito From b1a1071b568d6947dfa9579f4540906f3fe41223 Mon Sep 17 00:00:00 2001 From: Caleb Meier Date: Wed, 12 Oct 2016 12:55:27 -0700 Subject: [PATCH 3/4] Fluo Upgrade --- .../rya/api/client/accumulo/FluoITBase.java | 33 +++++----- .../apache/rya/indexing/pcj/fluo/ITBase.java | 62 +++++++++---------- .../pcj/fluo/api/CountStatementsIT.java | 6 +- .../pcj/fluo/integration/CreateDeleteIT.java | 26 +++----- 4 files changed, 57 insertions(+), 70 deletions(-) diff --git a/extras/indexing/src/test/java/mvm/rya/api/client/accumulo/FluoITBase.java b/extras/indexing/src/test/java/mvm/rya/api/client/accumulo/FluoITBase.java index 0dcc04ccb..cc50b90ac 100644 --- a/extras/indexing/src/test/java/mvm/rya/api/client/accumulo/FluoITBase.java +++ b/extras/indexing/src/test/java/mvm/rya/api/client/accumulo/FluoITBase.java @@ -55,14 +55,14 @@ import com.google.common.io.Files; -import io.fluo.api.client.FluoAdmin; -import io.fluo.api.client.FluoAdmin.AlreadyInitializedException; -import io.fluo.api.client.FluoAdmin.TableExistsException; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.FluoFactory; -import io.fluo.api.config.FluoConfiguration; -import io.fluo.api.config.ObserverConfiguration; -import io.fluo.api.mini.MiniFluo; +import org.apache.fluo.api.client.FluoAdmin; +import org.apache.fluo.api.client.FluoAdmin.AlreadyInitializedException; +import org.apache.fluo.api.client.FluoAdmin.TableExistsException; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.config.ObserverSpecification; +import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.client.RyaClientException; import mvm.rya.api.client.Install; @@ -224,17 +224,16 @@ protected Map makeExportParams() { */ protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExistsException { // Setup the observers that will be used by the Fluo PCJ Application. - final List observers = new ArrayList<>(); - observers.add(new ObserverConfiguration(TripleObserver.class.getName())); - observers.add(new ObserverConfiguration(StatementPatternObserver.class.getName())); - observers.add(new ObserverConfiguration(JoinObserver.class.getName())); - observers.add(new ObserverConfiguration(FilterObserver.class.getName())); + final List observers = new ArrayList<>(); + observers.add(new ObserverSpecification(TripleObserver.class.getName())); + observers.add(new ObserverSpecification(StatementPatternObserver.class.getName())); + observers.add(new ObserverSpecification(JoinObserver.class.getName())); + observers.add(new ObserverSpecification(FilterObserver.class.getName())); // Provide export parameters child test classes may provide to the // export observer. - final ObserverConfiguration exportObserverConfig = new ObserverConfiguration( - QueryResultObserver.class.getName()); - exportObserverConfig.setParameters(makeExportParams()); + final ObserverSpecification exportObserverConfig = new ObserverSpecification( + QueryResultObserver.class.getName(), makeExportParams()); observers.add(exportObserverConfig); // Configure how the mini fluo will run. @@ -252,7 +251,7 @@ protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExis config.addObservers(observers); FluoFactory.newAdmin(config).initialize( - new FluoAdmin.InitOpts().setClearTable(true).setClearZookeeper(true) ); + new FluoAdmin.InitializationOptions().setClearTable(true).setClearZookeeper(true) ); return FluoFactory.newMiniFluo(config); } diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java index 9e2b9c66d..baefa532e 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java @@ -25,8 +25,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; -import java.util.Map.Entry; import java.util.Set; import org.apache.accumulo.core.client.AccumuloException; @@ -37,6 +37,18 @@ import org.apache.accumulo.core.client.security.tokens.PasswordToken; import org.apache.accumulo.minicluster.MiniAccumuloCluster; import org.apache.accumulo.minicluster.MiniAccumuloConfig; +import org.apache.fluo.api.client.FluoAdmin; +import org.apache.fluo.api.client.FluoAdmin.AlreadyInitializedException; +import org.apache.fluo.api.client.FluoAdmin.TableExistsException; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.FluoFactory; +import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.scanner.ColumnScanner; +import org.apache.fluo.api.client.scanner.RowScanner; +import org.apache.fluo.api.config.FluoConfiguration; +import org.apache.fluo.api.config.ObserverSpecification; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.mini.MiniFluo; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.rya.indexing.pcj.fluo.app.export.rya.RyaExportParameters; @@ -63,19 +75,7 @@ import org.openrdf.sail.Sail; import com.google.common.io.Files; -import org.apache.fluo.api.client.FluoAdmin; -import org.apache.fluo.api.client.FluoAdmin.AlreadyInitializedException; -import org.apache.fluo.api.client.FluoAdmin.TableExistsException; -import org.apache.fluo.api.client.FluoClient; -import org.apache.fluo.api.client.FluoFactory; -import org.apache.fluo.api.client.Snapshot; -import org.apache.fluo.api.config.FluoConfiguration; -import org.apache.fluo.api.config.ObserverConfiguration; -import org.apache.fluo.api.config.ScannerConfiguration; -import org.apache.fluo.api.data.Bytes; -import org.apache.fluo.api.iterator.ColumnIterator; -import org.apache.fluo.api.iterator.RowIterator; -import org.apache.fluo.api.mini.MiniFluo; + import mvm.rya.accumulo.AccumuloRdfConfiguration; import mvm.rya.api.client.Install.InstallConfiguration; import mvm.rya.api.client.RyaClient; @@ -283,17 +283,13 @@ protected static Set getQueryBindingSetValues(final FluoClient fluoC final QueryMetadata queryMetadata = new FluoQueryMetadataDAO().readQueryMetadata(snapshot, queryId); final VariableOrder varOrder = queryMetadata.getVariableOrder(); - // Fetch the Binding Sets for the query. - final ScannerConfiguration scanConfig = new ScannerConfiguration(); - scanConfig.fetchColumn(FluoQueryColumns.QUERY_BINDING_SET.getFamily(), - FluoQueryColumns.QUERY_BINDING_SET.getQualifier()); - + RowScanner rscanner = snapshot.scanner().fetch(FluoQueryColumns.QUERY_BINDING_SET).byRow().build(); final BindingSetStringConverter converter = new BindingSetStringConverter(); - final RowIterator rowIter = snapshot.get(scanConfig); - while (rowIter.hasNext()) { - final Entry row = rowIter.next(); - final String bindingSetString = row.getValue().next().getValue().toString(); + Iterator iter = rscanner.iterator(); + while (iter.hasNext()) { + ColumnScanner scanner = iter.next(); + final String bindingSetString = scanner.getsRow(); final BindingSet bindingSet = converter.convert(bindingSetString, varOrder); bindingSets.add(bindingSet); } @@ -378,14 +374,11 @@ protected static AccumuloRdfConfiguration makeConfig(final String instanceName, */ protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExistsException { // Setup the observers that will be used by the Fluo PCJ Application. - final List observers = new ArrayList<>(); - observers.add(new ObserverConfiguration(TripleObserver.class.getName())); - observers.add(new ObserverConfiguration(StatementPatternObserver.class.getName())); - observers.add(new ObserverConfiguration(JoinObserver.class.getName())); - observers.add(new ObserverConfiguration(FilterObserver.class.getName())); - - // Configure the export observer to export new PCJ results to the mini accumulo cluster. - final ObserverConfiguration exportObserverConfig = new ObserverConfiguration(QueryResultObserver.class.getName()); + final List observers = new ArrayList<>(); + observers.add(new ObserverSpecification(TripleObserver.class.getName())); + observers.add(new ObserverSpecification(StatementPatternObserver.class.getName())); + observers.add(new ObserverSpecification(JoinObserver.class.getName())); + observers.add(new ObserverSpecification(FilterObserver.class.getName())); final HashMap exportParams = new HashMap<>(); final RyaExportParameters ryaParams = new RyaExportParameters(exportParams); @@ -395,8 +388,9 @@ protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExis ryaParams.setZookeeperServers(zookeepers); ryaParams.setExporterUsername(ITBase.ACCUMULO_USER); ryaParams.setExporterPassword(ITBase.ACCUMULO_PASSWORD); - - exportObserverConfig.setParameters(exportParams); + + // Configure the export observer to export new PCJ results to the mini accumulo cluster. + final ObserverSpecification exportObserverConfig = new ObserverSpecification(QueryResultObserver.class.getName(), exportParams); observers.add(exportObserverConfig); // Configure how the mini fluo will run. @@ -414,7 +408,7 @@ protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExis config.addObservers(observers); FluoFactory.newAdmin(config).initialize( - new FluoAdmin.InitOpts().setClearTable(true).setClearZookeeper(true) ); + new FluoAdmin.InitializationOptions().setClearTable(true).setClearZookeeper(true) ); return FluoFactory.newMiniFluo(config); } } diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java index 630b86d6d..f7d4ee11d 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/api/CountStatementsIT.java @@ -34,7 +34,7 @@ import org.apache.fluo.api.client.FluoAdmin.TableExistsException; import org.apache.fluo.api.client.FluoFactory; import org.apache.fluo.api.config.FluoConfiguration; -import org.apache.fluo.api.config.ObserverConfiguration; +import org.apache.fluo.api.config.ObserverSpecification; import org.apache.fluo.api.mini.MiniFluo; import mvm.rya.api.domain.RyaStatement; import mvm.rya.api.domain.RyaURI; @@ -53,7 +53,7 @@ public class CountStatementsIT extends ITBase { @Override protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExistsException { // Setup the observers that will be used by the Fluo PCJ Application. - final List observers = new ArrayList<>(); + final List observers = new ArrayList<>(); // Configure how the mini fluo will run. final FluoConfiguration config = new FluoConfiguration(); @@ -70,7 +70,7 @@ protected MiniFluo startMiniFluo() throws AlreadyInitializedException, TableExis config.addObservers(observers); FluoFactory.newAdmin(config).initialize( - new FluoAdmin.InitOpts().setClearTable(true).setClearZookeeper(true) ); + new FluoAdmin.InitializationOptions().setClearTable(true).setClearZookeeper(true) ); final MiniFluo miniFluo = FluoFactory.newMiniFluo(config); return miniFluo; } diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java index 0a1852b5e..d2ff98cbf 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java @@ -23,9 +23,14 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Map.Entry; import java.util.Set; +import org.apache.fluo.api.client.FluoClient; +import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.scanner.ColumnScanner; +import org.apache.fluo.api.client.scanner.RowScanner; +import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.Span; import org.apache.rya.indexing.pcj.fluo.ITBase; import org.apache.rya.indexing.pcj.fluo.api.CreatePcj; import org.apache.rya.indexing.pcj.fluo.api.DeletePcj; @@ -39,14 +44,6 @@ import com.google.common.collect.Sets; -import io.fluo.api.client.FluoClient; -import io.fluo.api.client.Snapshot; -import io.fluo.api.config.ScannerConfiguration; -import io.fluo.api.data.Bytes; -import io.fluo.api.data.Span; -import io.fluo.api.iterator.ColumnIterator; -import io.fluo.api.iterator.RowIterator; - public class CreateDeleteIT extends ITBase { /** @@ -109,15 +106,12 @@ public void historicResults() throws Exception { private List getFluoTableEntries(FluoClient fluoClient) { try (Snapshot snapshot = fluoClient.newSnapshot()) { List rows = new ArrayList<>(); + RowScanner rscanner = snapshot.scanner().over(Span.prefix("")).byRow().build(); - ScannerConfiguration sc1 = new ScannerConfiguration(); - sc1.setSpan(Span.prefix("")); - RowIterator iterator = snapshot.get(sc1); - - while (iterator.hasNext()) { - Entry row = iterator.next(); - rows.add(row.getKey()); + for(ColumnScanner cscanner: rscanner) { + rows.add(cscanner.getRow()); } + return rows; } } From 81d56cbda4c4662fe73a51e76dc4fab669036e9c Mon Sep 17 00:00:00 2001 From: Caleb Meier Date: Thu, 13 Oct 2016 08:48:08 -0700 Subject: [PATCH 4/4] Fixed Integration Tests for Fluo Update --- .../java/org/apache/rya/indexing/pcj/fluo/ITBase.java | 10 ++++++---- .../fluo/integration/RyaInputIncrementalUpdateIT.java | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java index baefa532e..a5288ec8a 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/ITBase.java @@ -43,11 +43,13 @@ import org.apache.fluo.api.client.FluoClient; import org.apache.fluo.api.client.FluoFactory; import org.apache.fluo.api.client.Snapshot; +import org.apache.fluo.api.client.scanner.CellScanner; import org.apache.fluo.api.client.scanner.ColumnScanner; import org.apache.fluo.api.client.scanner.RowScanner; import org.apache.fluo.api.config.FluoConfiguration; import org.apache.fluo.api.config.ObserverSpecification; import org.apache.fluo.api.data.Bytes; +import org.apache.fluo.api.data.RowColumnValue; import org.apache.fluo.api.mini.MiniFluo; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -283,13 +285,13 @@ protected static Set getQueryBindingSetValues(final FluoClient fluoC final QueryMetadata queryMetadata = new FluoQueryMetadataDAO().readQueryMetadata(snapshot, queryId); final VariableOrder varOrder = queryMetadata.getVariableOrder(); - RowScanner rscanner = snapshot.scanner().fetch(FluoQueryColumns.QUERY_BINDING_SET).byRow().build(); + CellScanner cellScanner = snapshot.scanner().fetch(FluoQueryColumns.QUERY_BINDING_SET).build(); final BindingSetStringConverter converter = new BindingSetStringConverter(); - Iterator iter = rscanner.iterator(); + Iterator iter = cellScanner.iterator(); + while (iter.hasNext()) { - ColumnScanner scanner = iter.next(); - final String bindingSetString = scanner.getsRow(); + final String bindingSetString = iter.next().getsValue(); final BindingSet bindingSet = converter.convert(bindingSetString, varOrder); bindingSets.add(bindingSet); } diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java index bc83eedb9..82f568e01 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/RyaInputIncrementalUpdateIT.java @@ -25,6 +25,7 @@ import org.apache.rya.indexing.pcj.fluo.ITBase; import org.apache.rya.indexing.pcj.fluo.api.CreatePcj; +import org.apache.rya.indexing.pcj.fluo.app.IncUpdateDAO; import org.apache.rya.indexing.pcj.storage.PrecomputedJoinStorage; import org.apache.rya.indexing.pcj.storage.accumulo.AccumuloPcjStorage; import org.apache.rya.indexing.pcj.update.PrecomputedJoinUpdater; @@ -101,7 +102,7 @@ public void streamResultsThroughRya() throws Exception { } fluo.waitForObservers(); - + final Set results = getQueryBindingSetValues(fluoClient, sparql); assertEquals(expected, results); }