From 33a04863581e5a875b2ec9bd23a488b3595640db Mon Sep 17 00:00:00 2001 From: "David W. Lotts" Date: Wed, 13 Dec 2017 17:34:55 -0500 Subject: [PATCH 01/13] RYA-104 most interactors complete. --- .../org/apache/rya/api/client/RyaClient.java | 12 +- .../mongodb/instance/MongoDetailsAdapter.java | 26 +-- .../MongoRyaInstanceDetailsRepository.java | 12 +- .../rya/api/client/mongo/MongoCommand.java | 65 ++++++ .../client/mongo/MongoConnectionDetails.java | 93 ++++++++ .../rya/api/client/mongo/MongoCreatePCJ.java | 58 +++++ .../rya/api/client/mongo/MongoDeletePCJ.java | 51 +++++ .../client/mongo/MongoGetInstanceDetails.java | 80 +++++++ .../rya/api/client/mongo/MongoInstall.java | 199 ++++++++++++++++++ .../api/client/mongo/MongoInstanceExists.java | 53 +++++ .../api/client/mongo/MongoListInstances.java | 68 ++++++ .../api/client/mongo/MongoLoadStatements.java | 29 +++ .../client/mongo/MongoLoadStatementsFile.java | 87 ++++++++ .../client/mongo/MongoRyaClientFactory.java | 66 ++++++ .../rya/api/client/mongo/MongoUninstall.java | 61 ++++++ .../mongo/MongoGetInstanceDetailsIT.java | 123 +++++++++++ .../rya/api/client/mongo/MongoInstallIT.java | 105 +++++++++ .../client/mongo/MongoInstanceExistsIT.java | 86 ++++++++ .../client/mongo/MongoListInstancesIT.java | 71 +++++++ .../mongo/MongoLoadStatementsFileIT.java | 132 ++++++++++++ .../api/client/mongo/MongoUninstallIT.java | 78 +++++++ 21 files changed, 1531 insertions(+), 24 deletions(-) create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java create mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java diff --git a/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java b/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java index 1278193d1..0a55edb3b 100644 --- a/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java +++ b/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java @@ -74,16 +74,16 @@ public RyaClient( this.createPeriodicPcj = createPeriodicPcj; this.deletePeriodicPcj = deletePeriodicPcj; this.listIncrementalQueries = listIncrementalQueries; - this.bactchUpdatePCJ = requireNonNull(batchUpdatePcj); + this.bactchUpdatePCJ = batchUpdatePcj; this.getInstanceDetails = requireNonNull(getInstanceDetails); this.instanceExists = requireNonNull(instanceExists); this.listInstances = requireNonNull(listInstances); - this.addUser = requireNonNull(addUser); - this.removeUser = requireNonNull(removeUser); + this.addUser = addUser; + this.removeUser = removeUser; this.uninstall = requireNonNull(uninstall); - this.loadStatements = requireNonNull(loadStatements); - this.loadStatementsFile = requireNonNull(loadStatementsFile); - this.executeSparqlQuery = requireNonNull(executeSparqlQuery); + this.loadStatements = loadStatements; // FIXME requireNonNull(loadStatements); + this.loadStatementsFile = loadStatementsFile; // FIXME requireNonNull(loadStatementsFile); + this.executeSparqlQuery = executeSparqlQuery; // FIXME requireNonNull(executeSparqlQuery); } /** diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoDetailsAdapter.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoDetailsAdapter.java index d11af760f..3b77a88ba 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoDetailsAdapter.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoDetailsAdapter.java @@ -24,9 +24,6 @@ import java.util.Date; import java.util.List; -import edu.umd.cs.findbugs.annotations.DefaultAnnotation; -import edu.umd.cs.findbugs.annotations.NonNull; - import org.apache.rya.api.instance.RyaDetails; import org.apache.rya.api.instance.RyaDetails.EntityCentricIndexDetails; import org.apache.rya.api.instance.RyaDetails.FreeTextIndexDetails; @@ -45,6 +42,9 @@ import com.mongodb.BasicDBObjectBuilder; import com.mongodb.DBObject; +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + /** * Serializes configuration details for use in Mongo. * The {@link DBObject} will look like: @@ -182,15 +182,17 @@ public static RyaDetails toRyaDetails(final DBObject mongoObj) throws MalformedR private static PCJIndexDetails.Builder getPCJIndexDetails(final BasicDBObject basicObj) { final BasicDBObject pcjIndexDBO = (BasicDBObject) basicObj.get(PCJ_DETAILS_KEY); - final PCJIndexDetails.Builder pcjBuilder = PCJIndexDetails.builder() - .setEnabled(pcjIndexDBO.getBoolean(PCJ_ENABLED_KEY)) - .setFluoDetails(new FluoDetails(pcjIndexDBO.getString(PCJ_FLUO_KEY))); - - final BasicDBList pcjs = (BasicDBList) pcjIndexDBO.get(PCJ_PCJS_KEY); - if(pcjs != null) { - for(int ii = 0; ii < pcjs.size(); ii++) { - final BasicDBObject pcj = (BasicDBObject) pcjs.get(ii); - pcjBuilder.addPCJDetails( toPCJDetails(pcj) ); + final PCJIndexDetails.Builder pcjBuilder = PCJIndexDetails.builder(); + if (!pcjIndexDBO.getBoolean(PCJ_ENABLED_KEY)) { + pcjBuilder.setEnabled(false); + } else { + pcjBuilder.setEnabled(true).setFluoDetails(new FluoDetails(pcjIndexDBO.getString(PCJ_FLUO_KEY))); + final BasicDBList pcjs = (BasicDBList) pcjIndexDBO.get(PCJ_PCJS_KEY); + if (pcjs != null) { + for (int ii = 0; ii < pcjs.size(); ii++) { + final BasicDBObject pcj = (BasicDBObject) pcjs.get(ii); + pcjBuilder.addPCJDetails(toPCJDetails(pcj)); + } } } return pcjBuilder; diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java index 4a961be4f..9bb558e86 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java @@ -22,8 +22,9 @@ import static com.google.common.base.Preconditions.checkNotNull; import static java.util.Objects.requireNonNull; -import edu.umd.cs.findbugs.annotations.DefaultAnnotation; -import edu.umd.cs.findbugs.annotations.NonNull; +import org.apache.rya.api.instance.RyaDetails; +import org.apache.rya.api.instance.RyaDetailsRepository; +import org.apache.rya.mongodb.instance.MongoDetailsAdapter.MalformedRyaDetailsException; import com.mongodb.BasicDBObject; import com.mongodb.DB; @@ -32,9 +33,8 @@ import com.mongodb.MongoClient; import com.mongodb.WriteResult; -import org.apache.rya.api.instance.RyaDetails; -import org.apache.rya.api.instance.RyaDetailsRepository; -import org.apache.rya.mongodb.instance.MongoDetailsAdapter.MalformedRyaDetailsException; +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; /** * An implementation of {@link RyaDetailsRepository} that stores a Rya @@ -42,7 +42,7 @@ */ @DefaultAnnotation(NonNull.class) public class MongoRyaInstanceDetailsRepository implements RyaDetailsRepository { - private static final String INSTANCE_DETAILS_COLLECTION_NAME = "instance_details"; + public static final String INSTANCE_DETAILS_COLLECTION_NAME = "instance_details"; private final DB db; private final String instanceName; diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java new file mode 100644 index 000000000..24bae951b --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java @@ -0,0 +1,65 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static java.util.Objects.requireNonNull; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An abstract class that holds onto Mongo access information. Extend this + * when implementing a command that interacts with Mongo. + */ +@DefaultAnnotation(NonNull.class) +public abstract class MongoCommand { + + private final MongoConnectionDetails connectionDetails; + private final MongoClient client; + + /** + * Constructs an instance of {@link MongoCommand}. + * + * Details about the values that were used to create the client. (not null) + * + * @param client + * - Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoCommand(final MongoConnectionDetails connectionDetails, final MongoClient client) { + this.connectionDetails = requireNonNull(connectionDetails); + this.client = requireNonNull(client); + } + + /** + * @return Details about the values that were used to create the connector to mongo. (not null) + */ + public MongoConnectionDetails getMongoConnectionDetails() { + return connectionDetails; + } + + /** + * @return Provides programatic access to the instance of Mongo that hosts Rya instance. + */ + public MongoClient getClient() { + return client; + } +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java new file mode 100644 index 000000000..16e045342 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -0,0 +1,93 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static java.util.Objects.requireNonNull; + +import org.apache.rya.accumulo.AccumuloRdfConfiguration; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; +public class MongoConnectionDetails { + private String username; + private char[] userPass; + private String instance; + private String host; + private String collectionName; + + public MongoConnectionDetails(String username, char[] userPass, String instance, String host, String ryaInstance) { + this.username = username; + this.userPass = userPass.clone(); + this.instance = instance; + this.host = host; + this.collectionName = requireNonNull(ryaInstance); + } + + /** + * @return the username + */ + public String getUsername() { + return this.username; + } + + /** + * @return the password + */ + public char[] getPassword() { + return this.userPass; + } + + /** + * @return the instance + */ + public String getInstance() { + return instance; + } + + /** + * @return the host AKA MongoInstance + */ + public String getHost() { + return host; + } + /** + * @return The Collection/Rya Prefix/Rya instance that was used to establish the connection. + */ + public String getCollectionName() { + return collectionName; + } + + /** + * + * @param ryaInstanceName + * - The Rya instance to connect to. + * @return Constructs a new {@link AccumuloRdfConfiguration} object with values from this object. + */ + public MongoDBRdfConfiguration build(final String ryaInstanceName) { + + // Note, we don't use the MongoDBRdfConfigurationBuilder here because it explicitly sets + // authorizations and visibilities to an empty string if they are not set on the builder. + // If they are null in the MongoRdfConfiguration object, it may do the right thing. + final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); + conf.setMongoInstance(host); + conf.setCollectionName(ryaInstanceName); + conf.setMongoUser(username); + conf.setMongoPassword(new String(userPass)); + return conf; + } + +} diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java new file mode 100644 index 000000000..30d0f1454 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import java.util.Set; + +import org.apache.rya.api.client.CreatePCJ; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClientException; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link CreatePCJ} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoCreatePCJ extends MongoCommand implements CreatePCJ { + + /** + * Constructs an instance of {@link MongoCreatePCJ}. + * + * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) + * @param client - Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoCreatePCJ(final MongoConnectionDetails connectionDetails, final MongoClient client) { + super(connectionDetails, client); + } + + @Override + public String createPCJ(final String instanceName, final String sparql) throws InstanceDoesNotExistException { + throw new UnsupportedOperationException("Mongo does not support PCJ indexing."); + } + + @Override + public String createPCJ(String instanceName, String sparql, Set strategies) throws InstanceDoesNotExistException, RyaClientException { + throw new UnsupportedOperationException("Mongo does not support PCJ indexing."); + } +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java new file mode 100644 index 000000000..6e16bf0e8 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import org.apache.rya.api.client.DeletePCJ; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClientException; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link DeletePCJ} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoDeletePCJ extends MongoCommand implements DeletePCJ { + + /** + * Constructs an instance of {@link MongoDeletePCJ}. + * + * @param connectionDetails Details about the values that were used to create the connector to the cluster. (not null) + * @param client Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoDeletePCJ(final MongoConnectionDetails connectionDetails, final MongoClient client) { + super(connectionDetails, client); + } + + @Override + public void deletePCJ(final String instanceName, final String pcjId) throws InstanceDoesNotExistException, RyaClientException { + throw new UnsupportedOperationException("Mongo does not support PCJ Indexing."); + } +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java new file mode 100644 index 000000000..1bfa97a1b --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static java.util.Objects.requireNonNull; + +import org.apache.rya.api.client.GetInstanceDetails; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.instance.RyaDetails; +import org.apache.rya.api.instance.RyaDetailsRepository; +import org.apache.rya.api.instance.RyaDetailsRepository.NotInitializedException; +import org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException; +import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; + +import com.google.common.base.Optional; +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link GetInstanceDetails} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoGetInstanceDetails extends MongoCommand implements GetInstanceDetails { + + private final InstanceExists instanceExists; + + /** + * Constructs an instance of {@link MongoGetInstanceDetails}. + * + * @param connectionDetails + * - Details about the values that were used to create the connector to the cluster. (not null) + * @param client + * - Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoGetInstanceDetails(final MongoConnectionDetails connectionDetails, final MongoClient client) { + super(connectionDetails, client); + instanceExists = new MongoInstanceExists(connectionDetails, client); + } + + @Override + public Optional getDetails(final String ryaInstanceName) throws InstanceDoesNotExistException, RyaClientException { + requireNonNull(ryaInstanceName); + + // Ensure the Rya instance exists. + if (!instanceExists.exists(ryaInstanceName)) { + throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", ryaInstanceName)); + } + + // If the instance has details, then return them. + final RyaDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getClient(), ryaInstanceName); + try { + return Optional.of(detailsRepo.getRyaInstanceDetails()); + } catch (final NotInitializedException e) { + return Optional.absent(); + } catch (final RyaDetailsRepositoryException e) { + throw new RyaClientException("Could not fetch the Rya instance's details.", e); + } + } +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java new file mode 100644 index 000000000..276653663 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java @@ -0,0 +1,199 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static java.util.Objects.requireNonNull; + +import java.util.Date; + +import org.apache.rya.api.RdfCloudTripleStoreConfiguration; +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.instance.RyaDetails; +import org.apache.rya.api.instance.RyaDetails.EntityCentricIndexDetails; +import org.apache.rya.api.instance.RyaDetails.FreeTextIndexDetails; +import org.apache.rya.api.instance.RyaDetails.JoinSelectivityDetails; +import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails; +import org.apache.rya.api.instance.RyaDetails.ProspectorDetails; +import org.apache.rya.api.instance.RyaDetails.TemporalIndexDetails; +import org.apache.rya.api.instance.RyaDetailsRepository; +import org.apache.rya.api.instance.RyaDetailsRepository.AlreadyInitializedException; +import org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException; +import org.apache.rya.api.layout.TablePrefixLayoutStrategy; +import org.apache.rya.api.persist.RyaDAOException; +import org.apache.rya.indexing.accumulo.ConfigUtils; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import org.apache.rya.mongodb.MongoDBRyaDAO; +import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; + +import com.google.common.base.Optional; +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link Install} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoInstall extends MongoCommand implements Install { + + private final InstanceExists instanceExists; + + /** + * Constructs an instance of {@link MongoInstall}. + * + * @param connectionDetails + * - Details about the values that were used to create the connector to the cluster. (not null) + * @param client + * - Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoInstall(final MongoConnectionDetails connectionDetails, final MongoClient client) { + super(connectionDetails, client); + instanceExists = new MongoInstanceExists(connectionDetails, client); + } + + @Override + public void install(final String instanceName, final InstallConfiguration installConfig) throws DuplicateInstanceNameException, RyaClientException { + requireNonNull(instanceName, "instanceName required."); + requireNonNull(installConfig, "installConfig required."); + + // Check to see if a Rya instance has already been installed with this name. + if (instanceExists.exists(instanceName)) { + throw new DuplicateInstanceNameException("An instance of Rya has already been installed to this Rya storage " + "with the name '" + instanceName + "'. Try again with a different name."); + } + + // Initialize the Rya Details table. + RyaDetails details; + try { + details = initializeRyaDetails(instanceName, installConfig); + } catch (final AlreadyInitializedException e) { + // This can only happen if somebody else installs an instance of Rya with the name between the check and now. + throw new DuplicateInstanceNameException("An instance of Rya has already been installed to this Rya storage "// + + "with the name '" + instanceName// + + "'. Try again with a different name."); + } catch (final RyaDetailsRepositoryException e) { + throw new RyaClientException("The RyaDetails couldn't be initialized. Details: " + e.getMessage(), e); + } + + // Initialize the rest of the tables used by the Rya instance. + final MongoDBRdfConfiguration ryaConfig = makeRyaConfig(getMongoConnectionDetails(), details); + try { + final MongoDBRyaDAO ryaDao = new MongoDBRyaDAO(ryaConfig, getClient()); + ryaDao.setConf(ryaConfig); + + final TablePrefixLayoutStrategy tls = new TablePrefixLayoutStrategy(); + tls.setTablePrefix(instanceName); + ryaConfig.setTableLayoutStrategy(tls); + + ryaDao.init(); + } catch (final RyaDAOException e) { + throw new RyaClientException("Could not initialize all of the tables for the new Rya instance. " // + + "This instance may be left in a bad state.", e); + } + } + + /** + * @return The version of the application as reported by the manifest. + */ + private String getVersion() { + return "" + this.getClass().getPackage().getImplementationVersion(); + } + + /** + * Initializes the {@link RyaDetails} and stores them for the new instance. + * + * @param instanceName + * - The name of the instance that is being created. (not null) + * @param installConfig + * - The instance's install configuration. (not null) + * @return The {@link RyaDetails} that were stored. + * @throws AlreadyInitializedException + * Could not be initialized because + * a table with this instance name has already exists and is holding the details. + * @throws RyaDetailsRepositoryException + * Something caused the initialization + * operation to fail. + */ + private RyaDetails initializeRyaDetails(final String instanceName, final InstallConfiguration installConfig) throws AlreadyInitializedException, RyaDetailsRepositoryException { + final RyaDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getClient(), instanceName); + + // Build the PCJ Index details. [not supported in mongo] + final PCJIndexDetails.Builder pcjDetailsBuilder = PCJIndexDetails.builder().setEnabled(false); + + final RyaDetails details = RyaDetails.builder() + // General Metadata + .setRyaInstanceName(instanceName).setRyaVersion(getVersion()) + + // Secondary Index Values + // FIXME .setGeoIndexDetails(new GeoIndexDetails(installConfig.isGeoIndexEnabled())) + .setTemporalIndexDetails(new TemporalIndexDetails(installConfig.isTemporalIndexEnabled())) // + .setFreeTextDetails(new FreeTextIndexDetails(installConfig.isFreeTextIndexEnabled()))// + .setEntityCentricIndexDetails(new EntityCentricIndexDetails(false))// not supported in mongo + .setPCJIndexDetails(pcjDetailsBuilder) + + // Statistics values. + .setProspectorDetails(new ProspectorDetails(Optional. absent()))// + .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional. absent()))// + .build(); + + // Initialize the table. + detailsRepo.initialize(details); + + return details; + } + + /** + * Builds a {@link MongoRdfConfiguration} object that will be used by the + * Rya DAO to initialize all of the tables it will need. + * + * @param connectionDetails + * - Indicates how to connect to Mongo. (not null) + * @param details + * - Indicates what needs to be installed. (not null) + * @return A Rya Configuration object that can be used to perform the install. + */ + private static MongoDBRdfConfiguration makeRyaConfig(final MongoConnectionDetails connectionDetails, final RyaDetails details) { + final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); + conf.setBoolean(ConfigUtils.USE_MONGO, true); + // The Rya Instance Name is used as a prefix for the index tables in Mongo. + conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, details.getRyaInstanceName()); + + // Enable the indexers that the instance is configured to use. + conf.set(ConfigUtils.USE_PCJ, "" + details.getPCJIndexDetails().isEnabled()); + // fixme conf.set(OptionalConfigUtils.USE_GEO, "" + details.getGeoIndexDetails().isEnabled() ); + conf.set(ConfigUtils.USE_FREETEXT, "" + details.getFreeTextIndexDetails().isEnabled()); + conf.set(ConfigUtils.USE_TEMPORAL, "" + details.getTemporalIndexDetails().isEnabled()); + + // Mongo does not support entity indexing. + conf.set(ConfigUtils.USE_ENTITY, "" + false); + + conf.set(ConfigUtils.CLOUDBASE_USER, connectionDetails.getUsername()); + conf.set(ConfigUtils.CLOUDBASE_PASSWORD, new String(connectionDetails.getPassword())); + conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, details.getRyaInstanceName()); + + // This initializes the living indexers that will be used by the application and + // caches them within the configuration object so that they may be used later. + ConfigUtils.setIndexers(conf); + + return conf; + } +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java new file mode 100644 index 000000000..256085657 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static java.util.Objects.requireNonNull; + +import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.RyaClientException; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link InstanceExists} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoInstanceExists extends MongoCommand implements InstanceExists { + + /** + * Constructs an insatnce of {@link MongoInstanceExists}. + * + * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) + * @param client - Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoInstanceExists(final MongoConnectionDetails connectionDetails, final MongoClient client) { + super(connectionDetails, client); + } + + @Override + public boolean exists(final String instanceName) { + requireNonNull( instanceName ); + return super.getClient().getDatabaseNames().contains(instanceName); + } +} diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java new file mode 100644 index 000000000..d301bf2c1 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +import org.apache.rya.api.client.ListInstances; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link ListInstances} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoListInstances extends MongoCommand implements ListInstances { + /** + * Constructs an instance of {@link MongoListInstances}. + * + * @param connectionDetails + * - Details about the values that were used to create the connector to the cluster. (not null) + * @param client + * - Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoListInstances(final MongoConnectionDetails connectionDetails, final MongoClient client) { + super(connectionDetails, client); + } + + @Override + public List listInstances() throws RyaClientException { + final MongoClient client = super.getClient(); + final List dbNames = client.getDatabaseNames(); + final List ryaInstances = new ArrayList<>(); + for (final String db : dbNames) { + final Set collNames = client.getDB(db).getCollectionNames(); + for (final String coll : collNames) { + if (coll.equals(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME)) { + ryaInstances.add(db); + break; + } + } + } + return ryaInstances; + } +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java new file mode 100644 index 000000000..844489da7 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java @@ -0,0 +1,29 @@ +package org.apache.rya.api.client.mongo; + +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.LoadStatements; +import org.apache.rya.api.client.RyaClientException; +import org.openrdf.model.Statement; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link LoadStatements} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoLoadStatements extends MongoCommand implements LoadStatements { + + public MongoLoadStatements(MongoConnectionDetails connectionDetails, MongoClient client) { + super(connectionDetails, client); + } + + @Override + public void loadStatements(String ryaInstanceName, Iterable statements) throws InstanceDoesNotExistException, RyaClientException { + // TODO Auto-generated method stub + + } + +} diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java new file mode 100644 index 000000000..bf525c1f0 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java @@ -0,0 +1,87 @@ +package org.apache.rya.api.client.mongo; + +import static java.util.Objects.requireNonNull; + +import java.io.IOException; +import java.nio.file.Path; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.log4j.Logger; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.LoadStatementsFile; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.persist.RyaDAOException; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import org.apache.rya.rdftriplestore.inference.InferenceEngineException; +import org.apache.rya.sail.config.RyaSailFactory; +import org.openrdf.repository.RepositoryException; +import org.openrdf.repository.sail.SailRepository; +import org.openrdf.repository.sail.SailRepositoryConnection; +import org.openrdf.rio.RDFFormat; +import org.openrdf.rio.RDFParseException; +import org.openrdf.sail.Sail; +import org.openrdf.sail.SailException; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link LoadStatementsFile} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoLoadStatementsFile extends MongoCommand implements LoadStatementsFile { + private static final Logger log = Logger.getLogger(MongoLoadStatementsFile.class); + + private final InstanceExists instanceExists; + + /** + * Constructs an instance of {@link MongoListInstances}. + * + * @param connectionDetails + * - Details to connect to the server. (not null) + * @param client + * - Provides programmatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoLoadStatementsFile(MongoConnectionDetails connectionDetails, MongoClient client) { + super(connectionDetails, client); + instanceExists = new MongoInstanceExists(connectionDetails, client); + } + + @Override + public void loadStatements(String ryaInstanceName, Path statementsFile, RDFFormat format) throws InstanceDoesNotExistException, RyaClientException { + requireNonNull(ryaInstanceName); + requireNonNull(statementsFile); + requireNonNull(format); + + // Ensure the Rya Instance exists. + if (!instanceExists.exists(ryaInstanceName)) { + throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", ryaInstanceName)); + } + + Sail sail = null; + SailRepository sailRepo = null; + SailRepositoryConnection sailRepoConn = null; + // Get a Sail object that is connected to the Rya instance. + final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName); + // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. + try { + sail = RyaSailFactory.getInstance(ryaConf); + } catch (SailException | AccumuloException | AccumuloSecurityException | RyaDAOException | InferenceEngineException e) { + throw new RyaClientException("While getting an sail instance.", e); + } + + // Load the file. + sailRepo = new SailRepository(sail); + try { + sailRepoConn = sailRepo.getConnection(); + sailRepoConn.add(statementsFile.toFile(), null, format); + } catch (RDFParseException | RepositoryException | IOException e) { + throw new RyaClientException("While getting a connection and adding statements from a file.", e); + } + } +} diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java new file mode 100644 index 000000000..4ea3487a9 --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static java.util.Objects.requireNonNull; + +import org.apache.rya.api.client.RyaClient; + +import com.mongodb.MongoClient; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * Constructs instance of {@link RyaClient} that are connected to instance of + * Rya hosted by Mongo clusters. + */ +@DefaultAnnotation(NonNull.class) +public class MongoRyaClientFactory { + + /** + * Initialize a set of {@link RyaClient} that will interact with an instance of + * Rya that is hosted by a MongoDB server. + * + * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) + * @param connector - The MongoDB connector the commands will use. (not null) + * @return The initialized commands. + */ + public static RyaClient build( + final MongoConnectionDetails connectionDetails, + final MongoClient connector) { + requireNonNull(connectionDetails); + requireNonNull(connector); + + // Build the RyaCommands option with the initialized commands. + return new RyaClient(// + new MongoInstall(connectionDetails, connector), // + new MongoCreatePCJ(connectionDetails, connector), // + new MongoDeletePCJ(connectionDetails, connector), // + null, null, null, null, + new MongoGetInstanceDetails(connectionDetails, connector), // + new MongoInstanceExists(connectionDetails, connector), // + new MongoListInstances(connectionDetails, connector), // + null, null, + new MongoUninstall(connectionDetails, connector), // + new MongoLoadStatements(connectionDetails, connector), // + new MongoLoadStatementsFile(connectionDetails, connector), // + null);// FIXME new MongoExecuteSparqlQuery(connectionDetails, connector)); + } +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java new file mode 100644 index 000000000..b3988b80e --- /dev/null +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.client.Uninstall; + +import com.mongodb.MongoClient; +import com.mongodb.MongoException; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * An Mongo implementation of the {@link Uninstall} command. + */ +@DefaultAnnotation(NonNull.class) +public class MongoUninstall extends MongoCommand implements Uninstall { + + /** + * Constructs an instance of {@link MongoUninstall}. + * + * @param connectionDetails + * - Details about the values that were used to create the connector to the cluster. (not null) + * @param connector + * - Provides programatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ + public MongoUninstall(final MongoConnectionDetails connectionDetails, final MongoClient client) { + super(connectionDetails, client); + } + + @Override + public void uninstall(final String ryaInstanceName) throws InstanceDoesNotExistException, RyaClientException { + try { + if (!getClient().getDatabaseNames().contains(ryaInstanceName)) { + throw new InstanceDoesNotExistException("The database '" + ryaInstanceName + "' does not exist."); + } + getClient().dropDatabase(ryaInstanceName); + } catch (final MongoException e) { + throw new RyaClientException("Failed to uninstall '" + ryaInstanceName + "' " + e.getLocalizedMessage(), e); + } + } +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java new file mode 100644 index 000000000..843ba42d4 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java @@ -0,0 +1,123 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; + +import java.util.Date; + +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.rya.api.client.GetInstanceDetails; +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.instance.RyaDetails; +import org.apache.rya.api.instance.RyaDetails.EntityCentricIndexDetails; +import org.apache.rya.api.instance.RyaDetails.FreeTextIndexDetails; +import org.apache.rya.api.instance.RyaDetails.JoinSelectivityDetails; +import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails; +import org.apache.rya.api.instance.RyaDetails.ProspectorDetails; +import org.apache.rya.api.instance.RyaDetails.TemporalIndexDetails; +import org.apache.rya.mongodb.MongoTestBase; +import org.junit.Test; + +import com.google.common.base.Optional; +import com.mongodb.BasicDBObject; +import com.mongodb.MongoException; + +/** + * Tests the methods of {@link MongoGetInstanceDetails}. + */ +public class MongoGetInstanceDetailsIT extends MongoTestBase { + + @Test + public void getDetails() throws MongoException, RyaClientException { + final String instanceName = "instance"; + // Install an instance of Rya. + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(true) + .setEnableEntityCentricIndex(true) + .setEnableFreeTextIndex(true) + .setEnableTemporalIndex(true) + .setEnableGeoIndex(true) + .setEnablePcjIndex(false) + .build(); + + final Install install = new MongoInstall(getConnectionDetails(), this.getMongoClient()); + install.install(instanceName, installConfig); + + // Verify the correct details were persisted. + final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), this.getMongoClient()); + final Optional details = getInstanceDetails.getDetails(instanceName); + + final RyaDetails expectedDetails = RyaDetails.builder() + .setRyaInstanceName(instanceName) + + // The version depends on how the test is packaged, so just grab whatever was stored. + .setRyaVersion( details.get().getRyaVersion() ) + + // FIXME .setGeoIndexDetails( new GeoIndexDetails(true) ) + .setTemporalIndexDetails(new TemporalIndexDetails(true) ) + .setFreeTextDetails( new FreeTextIndexDetails(true) ) + .setEntityCentricIndexDetails( new EntityCentricIndexDetails(false) ) + .setPCJIndexDetails( + PCJIndexDetails.builder() + .setEnabled(false)) + .setProspectorDetails( new ProspectorDetails(Optional.absent()) ) + .setJoinSelectivityDetails( new JoinSelectivityDetails(Optional.absent()) ) + .build(); + + assertEquals(expectedDetails, details.get()); + } + + @Test(expected = InstanceDoesNotExistException.class) + public void getDetails_instanceDoesNotExist() throws MongoException, RyaClientException { + final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), conf.getMongoClient()); + getInstanceDetails.getDetails("instance_name"); + } + + @Test + public void getDetails_instanceDoesNotHaveDetails() throws MongoException, TableExistsException, RyaClientException { + // Mimic a pre-details rya install. + final String instanceName = "instance_name"; + + this.getMongoClient().getDB(instanceName).createCollection("rya_triples", new BasicDBObject()); + + // Verify that the operation returns empty. + final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), this.getMongoClient()); + final Optional details = getInstanceDetails.getDetails(instanceName); + assertFalse( details.isPresent() ); + } + + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( + conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoDBName(), // aka instance + conf.getMongoInstance(), // aka hostname + conf.getCollectionName() + ); + return connectionDetails; + } +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java new file mode 100644 index 000000000..e9dae1c85 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; + +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.DuplicateInstanceNameException; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.client.accumulo.AccumuloInstall; +import org.apache.rya.mongodb.MongoTestBase; +import org.junit.Test; + +/** + * Integration tests the methods of {@link AccumuloInstall}. + */ +public class MongoInstallIT extends MongoTestBase { + + @Test + public void install() throws DuplicateInstanceNameException, RyaClientException { + // Install an instance of Rya. + final String instanceName = conf.getCollectionName(); + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(false) // + .setEnableEntityCentricIndex(false)// + .setEnableFreeTextIndex(false)// + .setEnableTemporalIndex(false)// + .setEnablePcjIndex(false)// + .setEnableGeoIndex(false)// + .setFluoPcjAppName("fluo_app_name")// + .build(); + + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( + conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoDBName(), // aka instance + conf.getMongoInstance(), // aka hostname + conf.getCollectionName() + ); + + // Check that the instance does not exist. + assertFalse("Instance should NOT exist yet.", this.getMongoClient().getDatabaseNames().contains(instanceName)); + final InstanceExists instanceExists = new MongoInstanceExists(connectionDetails, this.getMongoClient()); + assertFalse("Instance should NOT exist yet.", instanceExists.exists(instanceName)); + + final Install install = new MongoInstall(connectionDetails, this.getMongoClient()); + install.install(instanceName, installConfig); + + // Check that the instance exists. + assertTrue("Instance should exist.", this.getMongoClient().getDatabaseNames().contains(instanceName)); + List expected = Arrays.asList("instance_details", instanceName + "_triples"); + int count = 0; + String found = ""; + for (String collection : this.getMongoClient().getDatabase(instanceName).listCollectionNames()) + { + System.out.println("Collection names:" + collection); + count += expected.contains(collection) ? 1 : 0; + found += ", " + collection; + } + assertTrue("Tables missing from:" + expected + " actual:" + found, expected.size() == count); + assertTrue("Instance should exist.", instanceExists.exists(instanceName)); + } + + @Test(expected = DuplicateInstanceNameException.class) + public void install_alreadyExists() throws DuplicateInstanceNameException, RyaClientException { + // Install an instance of Rya. + final String instanceName = conf.getCollectionName(); + final InstallConfiguration installConfig = InstallConfiguration.builder().build(); + + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoDBName(), // aka instance + conf.getMongoInstance(), // aka hostname + conf.getCollectionName() + ); + + final Install install = new MongoInstall(connectionDetails, this.getMongoClient()); + install.install(instanceName, installConfig); + + // Install it again throws expected error. + install.install(instanceName, installConfig); + } +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java new file mode 100644 index 000000000..35568bcf9 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java @@ -0,0 +1,86 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.rya.api.client.accumulo.AccumuloInstanceExists; +import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; +import org.junit.Test; + +import com.mongodb.BasicDBObject; +import com.mongodb.MongoClient; +import com.mongodb.MongoException; + +/** + * Integration tests the methods of {@link AccumuloInstanceExists}. + */ +public class MongoInstanceExistsIT extends MongoTestBase { + + @Test + public void exists_ryaDetailsTable() throws MongoException, TableExistsException { + final MongoClient client = getMongoClient(); + + // Create the Rya instance's Rya details collection. + final String instanceName = "test_instance_"; + client.getDB(instanceName).createCollection(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME, new BasicDBObject()); + + // Verify the command reports the instance exists. + final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), client); + assertTrue( instanceExists.exists(instanceName) ); + } + + @Test + public void exists_dataTables() throws MongoException, TableExistsException { + final MongoClient client = getMongoClient(); + + // Create the Rya instance's Rya triples collection. + final String instanceName = "test_instance_"; + client.getDB(instanceName).createCollection("rya_triples", new BasicDBObject()); + + // Verify the command reports the instance exists. + final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), client); + assertTrue( instanceExists.exists(instanceName) ); + } + + @Test + public void doesNotExist() throws MongoException { + // Verify the command reports the instance does not exists. + final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), getMongoClient()); + assertFalse( instanceExists.exists("some_instance") ); + } + + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoDBName(), // aka instance + conf.getMongoInstance(), // aka hostname + conf.getCollectionName() + ); + return connectionDetails; + } + + +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java new file mode 100644 index 000000000..7d5a2fe74 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import java.util.List; + +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.DuplicateInstanceNameException; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.ListInstances; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.mongodb.MongoTestBase; +import org.junit.Test; + +import com.beust.jcommander.internal.Lists; +import com.mongodb.MongoException; + +/** + * Integration tests the methods of {@link MongoListInstances}. + */ +public class MongoListInstancesIT extends MongoTestBase { + + @Test + public void listInstances_hasRyaDetailsTable() throws MongoException, DuplicateInstanceNameException, RyaClientException { + // Install a few instances of Rya using the install command. + final Install install = new MongoInstall(getConnectionDetails(), getMongoClient()); + install.install("instance1_", InstallConfiguration.builder().build()); + install.install("instance2_", InstallConfiguration.builder().build()); + install.install("instance3_", InstallConfiguration.builder().build()); + + // Fetch the list and verify it matches what is expected. + final ListInstances listInstances = new MongoListInstances(getConnectionDetails(), getMongoClient()); + final List instances = listInstances.listInstances(); + Collections.sort(instances); + + final List expected = Lists.newArrayList("instance1_", "instance2_", "instance3_"); + assertEquals(expected, instances); + } + + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoDBName(), // aka instance + conf.getMongoInstance(), // aka hostname + conf.getCollectionName() + ); + return connectionDetails; + } +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java new file mode 100644 index 000000000..6794d865c --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import static org.junit.Assert.assertEquals; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClient; +import org.apache.rya.mongodb.MongoTestBase; +import org.bson.Document; +import org.junit.Test; +import org.openrdf.model.Statement; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.ValueFactoryImpl; +import org.openrdf.rio.RDFFormat; + +import com.mongodb.client.MongoCursor; + +/** + * Integration tests the methods of {@link MongoLoadStatementsFile}. + */ +public class MongoLoadStatementsFileIT extends MongoTestBase { + + @Test(expected = InstanceDoesNotExistException.class) + public void instanceDoesNotExist() throws Exception { + + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); + ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getInstance(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); + } + + @Test + public void loadTurtleFile() throws Exception { + // Install an instance of Rya. + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(false) + .setEnableEntityCentricIndex(false) + .setEnableFreeTextIndex(false) + .setEnableTemporalIndex(false) + .setEnablePcjIndex(false) + .setEnableGeoIndex(false) + .build(); + MongoConnectionDetails connectionDetails = getConnectionDetails(); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final Install install = ryaClient.getInstall(); + install.install(connectionDetails.getInstance(), installConfig); + + // Load the test statement file. + ryaClient.getLoadStatementsFile().loadStatements( // + connectionDetails.getInstance(), // + Paths.get("src/test/resources/example.ttl"), // + RDFFormat.TURTLE); + + // Verify that the statements were loaded. + final ValueFactory vf = new ValueFactoryImpl(); + + final List expected = new ArrayList<>(); + expected.add( vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob")) ); + expected.add( vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie")) ); + expected.add( vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream")) ); + + final List statements = new ArrayList<>(); + MongoCursor x = getRyaCollection().find().iterator(); + System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); + while (x.hasNext()) { + Document y = x.next(); + System.out.println("getRyaCollection()=" + y); + } + assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); + // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); + // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); + // final Iterator> it = scanner.iterator(); + // while(it.hasNext()) { + // final Entry next = it.next(); + // + // final Key key = next.getKey(); + // final byte[] row = key.getRow().getBytes(); + // final byte[] columnFamily = key.getColumnFamily().getBytes(); + // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); + // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); + // + // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); + // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); + // + // // Filter out the rya version statement if it is present. + // if(!isRyaMetadataStatement(vf, statement)) { + // statements.add( statement ); + // } + // } + // + // assertEquals(expected, statements); + } + + private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { + return statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#version") ) || + statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#rts") ); + } + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(// + conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoDBName(), // aka instance + conf.getMongoInstance(), // aka hostname + conf.getCollectionName() + ); + return connectionDetails; + } +} diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java new file mode 100644 index 000000000..523b4b260 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.client.Uninstall; +import org.apache.rya.mongodb.MongoTestBase; +import org.junit.Assert; +import org.junit.Test; + +import com.mongodb.MongoException; + +/** + * Integration tests the methods of {@link MongoInstall}. + */ +public class MongoUninstallIT extends MongoTestBase { + + @Test + public void uninstall() throws MongoException, RyaClientException { + // Install an instance of Rya. + final String instanceName = "testInstance_"; + final InstallConfiguration installConfig = InstallConfiguration.builder().setEnableTableHashPrefix(true).setEnableEntityCentricIndex(true).setEnableFreeTextIndex(true).setEnableTemporalIndex(true).setEnablePcjIndex(true).setEnableGeoIndex(true).setFluoPcjAppName("fluo_app_name").build(); + + final Install install = new MongoInstall(getConnectionDetails(), conf.getMongoClient()); + install.install(instanceName, installConfig); + + // Uninstall the instance + final Uninstall uninstall = new MongoUninstall(getConnectionDetails(), conf.getMongoClient()); + uninstall.uninstall(instanceName); + + // Check that the instance no longer exists. + final InstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), conf.getMongoClient()); + Assert.assertFalse(instanceExists.exists(instanceName)); + } + + @Test(expected = InstanceDoesNotExistException.class) + public void uninstall_instanceDoesNotExists() throws MongoException, RyaClientException { + // Install an instance of Rya. + final String instanceName = "testInstance_"; + + // Uninstall the instance + final Uninstall uninstall = new MongoUninstall(getConnectionDetails(), conf.getMongoClient()); + uninstall.uninstall(instanceName); + } + + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoDBName(), // aka instance + conf.getMongoInstance(), // aka hostname + conf.getCollectionName() + ); + return connectionDetails; + } +} \ No newline at end of file From e60da4c3fc589e3e65a22d4c3d7cb75c72eb631f Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Wed, 13 Dec 2017 18:14:40 -0500 Subject: [PATCH 02/13] RYA-104 Updated the RyaClient to optionally include features that are required for the Accumulo Client, but not the Mongo Client. --- .../org/apache/rya/api/client/Install.java | 4 +- .../org/apache/rya/api/client/RyaClient.java | 68 ++++--- .../accumulo/AccumuloRyaClientFactory.java | 18 +- .../rya/api/client/mongo/MongoCommand.java | 11 +- .../client/mongo/MongoConnectionDetails.java | 87 ++++---- .../rya/api/client/mongo/MongoCreatePCJ.java | 58 ------ ...ePCJ.java => MongoExecuteSparqlQuery.java} | 25 ++- .../client/mongo/MongoGetInstanceDetails.java | 7 +- .../rya/api/client/mongo/MongoInstall.java | 101 +++++----- .../api/client/mongo/MongoInstanceExists.java | 13 +- .../api/client/mongo/MongoListInstances.java | 18 +- .../client/mongo/MongoRyaClientFactory.java | 40 ++-- .../rya/api/client/mongo/MongoUninstall.java | 13 +- .../client/accumulo/AccumuloAddUserIT.java | 18 +- .../accumulo/AccumuloBatchUpdatePCJIT.java | 2 +- .../client/accumulo/AccumuloRemoveUserIT.java | 18 +- .../mongo/MongoExecuteSparqlQueryIT.java | 26 +++ .../mongo/MongoGetInstanceDetailsIT.java | 28 +-- .../rya/api/client/mongo/MongoInstallIT.java | 65 +++--- .../client/mongo/MongoInstanceExistsIT.java | 22 +- .../client/mongo/MongoListInstancesIT.java | 12 +- .../mongo/MongoLoadStatementsFileIT.java | 190 ++++++++---------- .../client/mongo/MongoLoadStatementsIT.java | 26 +++ .../api/client/mongo/MongoUninstallIT.java | 25 ++- .../periodic/KafkaLatencyBenchmark.java | 4 +- .../benchmark/query/QueryBenchmarkRunIT.java | 2 +- .../pcj/fluo/integration/CreateDeleteIT.java | 2 +- .../pcj/fluo/integration/QueryIT.java | 178 ++++++++-------- .../pcj/fluo/visibility/PcjVisibilityIT.java | 2 +- .../pcj/fluo/test/base/KafkaExportITBase.java | 4 +- .../pcj/functions/geo/GeoFunctionsIT.java | 2 +- .../apache/rya/shell/RyaAdminCommands.java | 36 ++-- .../rya/shell/RyaAdminCommandsTest.java | 22 +- 33 files changed, 573 insertions(+), 574 deletions(-) delete mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java rename extras/indexing/src/main/java/org/apache/rya/api/client/mongo/{MongoDeletePCJ.java => MongoExecuteSparqlQuery.java} (58%) create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java create mode 100644 extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java diff --git a/common/rya.api/src/main/java/org/apache/rya/api/client/Install.java b/common/rya.api/src/main/java/org/apache/rya/api/client/Install.java index fd754ead4..a716b6d47 100644 --- a/common/rya.api/src/main/java/org/apache/rya/api/client/Install.java +++ b/common/rya.api/src/main/java/org/apache/rya/api/client/Install.java @@ -38,13 +38,13 @@ public interface Install { /** * Install a new instance of Rya. * - * @param instanceName - Indicates the name of the Rya instance to install. (not null) + * @param ryaInstanceName - Indicates the name of the Rya instance to install. (not null) * @param installConfig - Configures how the Rya instance will operate. The * instance name that is in this variable must match the {@code instanceName}. (not null) * @throws DuplicateInstanceNameException A Rya instance already exists for the provided name. * @throws RyaClientException Something caused the command to fail. */ - public void install(final String instanceName, final InstallConfiguration installConfig) throws DuplicateInstanceNameException, RyaClientException; + public void install(final String ryaInstanceName, final InstallConfiguration installConfig) throws DuplicateInstanceNameException, RyaClientException; /** * A Rya instance already exists for the provided name. diff --git a/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java b/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java index 0a55edb3b..de0b542ff 100644 --- a/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java +++ b/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java @@ -20,6 +20,8 @@ import static java.util.Objects.requireNonNull; +import java.util.Optional; + import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; import net.jcip.annotations.Immutable; @@ -32,17 +34,17 @@ public class RyaClient { // Administrative functions. private final Install install; - private final CreatePCJ createPcj; - private final DeletePCJ deletePcj; - private final CreatePeriodicPCJ createPeriodicPcj; - private final DeletePeriodicPCJ deletePeriodicPcj; - private final ListIncrementalQueries listIncrementalQueries; - private final BatchUpdatePCJ bactchUpdatePCJ; + private final Optional createPcj; + private final Optional deletePcj; + private final Optional createPeriodicPcj; + private final Optional deletePeriodicPcj; + private final Optional listIncrementalQueries; + private final Optional bactchUpdatePCJ; private final GetInstanceDetails getInstanceDetails; private final InstanceExists instanceExists; private final ListInstances listInstances; - private final AddUser addUser; - private final RemoveUser removeUser; + private final Optional addUser; + private final Optional removeUser; private final Uninstall uninstall; private final LoadStatements loadStatements; private final LoadStatementsFile loadStatementsFile; @@ -53,17 +55,17 @@ public class RyaClient { */ public RyaClient( final Install install, - final CreatePCJ createPcj, - final DeletePCJ deletePcj, - final CreatePeriodicPCJ createPeriodicPcj, - final DeletePeriodicPCJ deletePeriodicPcj, - final ListIncrementalQueries listIncrementalQueries, - final BatchUpdatePCJ batchUpdatePcj, + final Optional createPcj, + final Optional deletePcj, + final Optional createPeriodicPcj, + final Optional deletePeriodicPcj, + final Optional listIncrementalQueries, + final Optional batchUpdatePcj, final GetInstanceDetails getInstanceDetails, final InstanceExists instanceExists, final ListInstances listInstances, - final AddUser addUser, - final RemoveUser removeUser, + final Optional addUser, + final Optional removeUser, final Uninstall uninstall, final LoadStatements loadStatements, final LoadStatementsFile loadStatementsFile, @@ -71,19 +73,19 @@ public RyaClient( this.install = requireNonNull(install); this.createPcj = requireNonNull(createPcj); this.deletePcj = requireNonNull(deletePcj); - this.createPeriodicPcj = createPeriodicPcj; - this.deletePeriodicPcj = deletePeriodicPcj; - this.listIncrementalQueries = listIncrementalQueries; - this.bactchUpdatePCJ = batchUpdatePcj; + this.createPeriodicPcj = requireNonNull(createPeriodicPcj); + this.deletePeriodicPcj = requireNonNull(deletePeriodicPcj); + this.listIncrementalQueries = requireNonNull(listIncrementalQueries); + this.bactchUpdatePCJ = requireNonNull(batchUpdatePcj); this.getInstanceDetails = requireNonNull(getInstanceDetails); this.instanceExists = requireNonNull(instanceExists); this.listInstances = requireNonNull(listInstances); - this.addUser = addUser; - this.removeUser = removeUser; + this.addUser = requireNonNull(addUser); + this.removeUser = requireNonNull(removeUser); this.uninstall = requireNonNull(uninstall); - this.loadStatements = loadStatements; // FIXME requireNonNull(loadStatements); - this.loadStatementsFile = loadStatementsFile; // FIXME requireNonNull(loadStatementsFile); - this.executeSparqlQuery = executeSparqlQuery; // FIXME requireNonNull(executeSparqlQuery); + this.loadStatements = loadStatements; + this.loadStatementsFile = loadStatementsFile; + this.executeSparqlQuery = executeSparqlQuery; } /** @@ -97,7 +99,7 @@ public Install getInstall() { * @return An instance of {@link CreatePCJ} that is connected to a Rya storage * if the Rya instance supports PCJ indexing. */ - public CreatePCJ getCreatePCJ() { + public Optional getCreatePCJ() { return createPcj; } @@ -105,21 +107,21 @@ public CreatePCJ getCreatePCJ() { * @return An instance of {@link DeletePCJ} that is connected to a Rya storage * if the Rya instance supports PCJ indexing. */ - public DeletePCJ getDeletePCJ() { + public Optional getDeletePCJ() { return deletePcj; } /** * @return An instance of {@link CreatePeridodicPCJ} that is connected to a Rya Periodic Storage */ - public CreatePeriodicPCJ getCreatePeriodicPCJ() { + public Optional getCreatePeriodicPCJ() { return createPeriodicPcj; } /** * @return An instance of {@link DeletePeriodicPCJ} that is connected to a Rya Periodic Storage */ - public DeletePeriodicPCJ getDeletePeriodicPCJ() { + public Optional getDeletePeriodicPCJ() { return deletePeriodicPcj; } @@ -127,7 +129,7 @@ public DeletePeriodicPCJ getDeletePeriodicPCJ() { * @return An instance of {@link ListIncrementalQueries} for displaying queries that are incrementallly * maintained by the Rya instance */ - public ListIncrementalQueries getListIncrementalQueries() { + public Optional getListIncrementalQueries() { return listIncrementalQueries; } @@ -135,7 +137,7 @@ public ListIncrementalQueries getListIncrementalQueries() { * @return An instance of {@link BatchUpdatePCJ} that is connect to a Rya storage * if the Rya instance supports PCJ indexing. */ - public BatchUpdatePCJ getBatchUpdatePCJ() { + public Optional getBatchUpdatePCJ() { return bactchUpdatePCJ; } @@ -163,14 +165,14 @@ public InstanceExists getInstanceExists() { /** * @return An instance of {@link AddUser} that is connected to a Rya storage. */ - public AddUser getAddUser() { + public Optional getAddUser() { return addUser; } /** * @return An instance of {@link DeleteUser} that is connected to a Rya storage. */ - public RemoveUser getRemoveUser() { + public Optional getRemoveUser() { return removeUser; } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloRyaClientFactory.java b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloRyaClientFactory.java index fcc712c11..f9df491fc 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloRyaClientFactory.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/accumulo/AccumuloRyaClientFactory.java @@ -20,6 +20,8 @@ import static java.util.Objects.requireNonNull; +import java.util.Optional; + import org.apache.accumulo.core.client.Connector; import org.apache.rya.api.client.RyaClient; @@ -50,17 +52,17 @@ public static RyaClient build( // Build the RyaCommands option with the initialized commands. return new RyaClient( new AccumuloInstall(connectionDetails, connector), - new AccumuloCreatePCJ(connectionDetails, connector), - new AccumuloDeletePCJ(connectionDetails, connector), - new AccumuloCreatePeriodicPCJ(connectionDetails, connector), - new AccumuloDeletePeriodicPCJ(connectionDetails, connector), - new AccumuloListIncrementalQueries(connectionDetails, connector), - new AccumuloBatchUpdatePCJ(connectionDetails, connector), + Optional.of(new AccumuloCreatePCJ(connectionDetails, connector)), + Optional.of(new AccumuloDeletePCJ(connectionDetails, connector)), + Optional.of(new AccumuloCreatePeriodicPCJ(connectionDetails, connector)), + Optional.of(new AccumuloDeletePeriodicPCJ(connectionDetails, connector)), + Optional.of(new AccumuloListIncrementalQueries(connectionDetails, connector)), + Optional.of(new AccumuloBatchUpdatePCJ(connectionDetails, connector)), new AccumuloGetInstanceDetails(connectionDetails, connector), new AccumuloInstanceExists(connectionDetails, connector), new AccumuloListInstances(connectionDetails, connector), - new AccumuloAddUser(connectionDetails, connector), - new AccumuloRemoveUser(connectionDetails, connector), + Optional.of(new AccumuloAddUser(connectionDetails, connector)), + Optional.of(new AccumuloRemoveUser(connectionDetails, connector)), new AccumuloUninstall(connectionDetails, connector), new AccumuloLoadStatements(connectionDetails, connector), new AccumuloLoadStatementsFile(connectionDetails, connector), diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java index 24bae951b..443fec789 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java @@ -38,11 +38,8 @@ public abstract class MongoCommand { /** * Constructs an instance of {@link MongoCommand}. * - * Details about the values that were used to create the client. (not null) - * - * @param client - * - Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param connectionDetails - Details about the values that were used to create the client. (not null) + * @param client - Provides programmatic access to the instance of Mongo that hosts Rya instance. (not null) */ public MongoCommand(final MongoConnectionDetails connectionDetails, final MongoClient client) { this.connectionDetails = requireNonNull(connectionDetails); @@ -50,14 +47,14 @@ public MongoCommand(final MongoConnectionDetails connectionDetails, final MongoC } /** - * @return Details about the values that were used to create the connector to mongo. (not null) + * @return Details about the values that were used to create the connector to mongo. */ public MongoConnectionDetails getMongoConnectionDetails() { return connectionDetails; } /** - * @return Provides programatic access to the instance of Mongo that hosts Rya instance. + * @param client - Provides programmatic access to the instance of Mongo that hosts Rya instance. */ public MongoClient getClient() { return client; diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index 16e045342..f05b38d03 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -20,74 +20,93 @@ import static java.util.Objects.requireNonNull; -import org.apache.rya.accumulo.AccumuloRdfConfiguration; +import org.apache.rya.api.RdfCloudTripleStoreConfiguration; +import org.apache.rya.indexing.accumulo.ConfigUtils; import org.apache.rya.mongodb.MongoDBRdfConfiguration; + +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; + +/** + * The information the shell used to connect to Mongo DB. + */ +@DefaultAnnotation(NonNull.class) public class MongoConnectionDetails { - private String username; - private char[] userPass; - private String instance; - private String host; - private String collectionName; - public MongoConnectionDetails(String username, char[] userPass, String instance, String host, String ryaInstance) { - this.username = username; - this.userPass = userPass.clone(); - this.instance = instance; - this.host = host; - this.collectionName = requireNonNull(ryaInstance); + private final String username; + private final char[] userPass; + private final String hostname; + private final int port; + + /** + * Constructs an instance of {@link MongoConnectionDetails}. + * + * @param username - The username that was used to establish the connection. (not null) + * @param password - The password that was used to establish the connection. (not null) + * @param hostname - The hostname of the Mongo DB that was connected to. (not null) + * @param port - The port of the Mongo DB that was connected to. (not null) + */ + public MongoConnectionDetails( + final String username, + final char[] userPass, + final String hostname, + final int port) { + this.username = requireNonNull(username); + this.userPass = requireNonNull(userPass); + this.hostname = requireNonNull(hostname); + this.port = port; } /** - * @return the username + * @return The username that was used to establish the connection. */ public String getUsername() { return this.username; } /** - * @return the password + * @return The password that was used to establish the connection. */ public char[] getPassword() { return this.userPass; } /** - * @return the instance + * @return The hostname of the Mongo DB that was connected to. */ - public String getInstance() { - return instance; + public String getHostname() { + return hostname; } /** - * @return the host AKA MongoInstance + * @return The port of the Mongo DB that was connected to. */ - public String getHost() { - return host; - } - /** - * @return The Collection/Rya Prefix/Rya instance that was used to establish the connection. - */ - public String getCollectionName() { - return collectionName; + public int getPort() { + return port; } /** + * Create a {@link MongoDBRdfConfiguration} that is using this object's values. * - * @param ryaInstanceName - * - The Rya instance to connect to. - * @return Constructs a new {@link AccumuloRdfConfiguration} object with values from this object. + * @param ryaInstanceName - The Rya instance to connect to. + * @return Constructs a new {@link MongoDBRdfConfiguration} object with values from this object. */ public MongoDBRdfConfiguration build(final String ryaInstanceName) { - // Note, we don't use the MongoDBRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the MongoRdfConfiguration object, it may do the right thing. final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); - conf.setMongoInstance(host); - conf.setCollectionName(ryaInstanceName); + conf.setBoolean(ConfigUtils.USE_MONGO, true); + conf.setMongoInstance(hostname); + conf.setMongoPort("" + port); conf.setMongoUser(username); conf.setMongoPassword(new String(userPass)); + conf.setMongoDBName(ryaInstanceName); + + // Both of these are ways to configure the collection prefixes. + conf.setCollectionName(ryaInstanceName); + conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, ryaInstanceName); + return conf; } - -} +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java deleted file mode 100644 index 30d0f1454..000000000 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCreatePCJ.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.api.client.mongo; - -import java.util.Set; - -import org.apache.rya.api.client.CreatePCJ; -import org.apache.rya.api.client.InstanceDoesNotExistException; -import org.apache.rya.api.client.RyaClientException; - -import com.mongodb.MongoClient; - -import edu.umd.cs.findbugs.annotations.DefaultAnnotation; -import edu.umd.cs.findbugs.annotations.NonNull; - -/** - * An Mongo implementation of the {@link CreatePCJ} command. - */ -@DefaultAnnotation(NonNull.class) -public class MongoCreatePCJ extends MongoCommand implements CreatePCJ { - - /** - * Constructs an instance of {@link MongoCreatePCJ}. - * - * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param client - Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) - */ - public MongoCreatePCJ(final MongoConnectionDetails connectionDetails, final MongoClient client) { - super(connectionDetails, client); - } - - @Override - public String createPCJ(final String instanceName, final String sparql) throws InstanceDoesNotExistException { - throw new UnsupportedOperationException("Mongo does not support PCJ indexing."); - } - - @Override - public String createPCJ(String instanceName, String sparql, Set strategies) throws InstanceDoesNotExistException, RyaClientException { - throw new UnsupportedOperationException("Mongo does not support PCJ indexing."); - } -} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java similarity index 58% rename from extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java rename to extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java index 6e16bf0e8..f51d42dbc 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoDeletePCJ.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java @@ -18,7 +18,9 @@ */ package org.apache.rya.api.client.mongo; -import org.apache.rya.api.client.DeletePCJ; +import static java.util.Objects.requireNonNull; + +import org.apache.rya.api.client.ExecuteSparqlQuery; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.RyaClientException; @@ -28,24 +30,27 @@ import edu.umd.cs.findbugs.annotations.NonNull; /** - * An Mongo implementation of the {@link DeletePCJ} command. + * TODO impl, test, doc */ @DefaultAnnotation(NonNull.class) -public class MongoDeletePCJ extends MongoCommand implements DeletePCJ { +public class MongoExecuteSparqlQuery extends MongoCommand implements ExecuteSparqlQuery { /** - * Constructs an instance of {@link MongoDeletePCJ}. + * Constructs an instance of {@link }. * - * @param connectionDetails Details about the values that were used to create the connector to the cluster. (not null) - * @param client Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param connectionDetails - Details about the values that were used to create the client. (not null) + * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ - public MongoDeletePCJ(final MongoConnectionDetails connectionDetails, final MongoClient client) { + public MongoExecuteSparqlQuery(final MongoConnectionDetails connectionDetails, final MongoClient client) { super(connectionDetails, client); } @Override - public void deletePCJ(final String instanceName, final String pcjId) throws InstanceDoesNotExistException, RyaClientException { - throw new UnsupportedOperationException("Mongo does not support PCJ Indexing."); + public String executeSparqlQuery(final String ryaInstanceName, final String sparqlQuery) throws InstanceDoesNotExistException, RyaClientException { + requireNonNull(ryaInstanceName); + requireNonNull(sparqlQuery); + + // TODO Auto-generated method stub + return null; } } \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java index 1bfa97a1b..b61e107c9 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java @@ -47,11 +47,8 @@ public class MongoGetInstanceDetails extends MongoCommand implements GetInstance /** * Constructs an instance of {@link MongoGetInstanceDetails}. * - * @param connectionDetails - * - Details about the values that were used to create the connector to the cluster. (not null) - * @param client - * - Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) + * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ public MongoGetInstanceDetails(final MongoConnectionDetails connectionDetails, final MongoClient client) { super(connectionDetails, client); diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java index 276653663..bcd79f6b7 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java @@ -22,7 +22,6 @@ import java.util.Date; -import org.apache.rya.api.RdfCloudTripleStoreConfiguration; import org.apache.rya.api.client.Install; import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.RyaClientException; @@ -42,6 +41,8 @@ import org.apache.rya.mongodb.MongoDBRdfConfiguration; import org.apache.rya.mongodb.MongoDBRyaDAO; import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.google.common.base.Optional; import com.mongodb.MongoClient; @@ -55,16 +56,15 @@ @DefaultAnnotation(NonNull.class) public class MongoInstall extends MongoCommand implements Install { + private static final Logger log = LoggerFactory.getLogger(MongoInstall.class); + private final InstanceExists instanceExists; /** * Constructs an instance of {@link MongoInstall}. * - * @param connectionDetails - * - Details about the values that were used to create the connector to the cluster. (not null) - * @param client - * - Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) + * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ public MongoInstall(final MongoConnectionDetails connectionDetails, final MongoClient client) { super(connectionDetails, client); @@ -121,39 +121,40 @@ private String getVersion() { /** * Initializes the {@link RyaDetails} and stores them for the new instance. * - * @param instanceName - * - The name of the instance that is being created. (not null) - * @param installConfig - * - The instance's install configuration. (not null) + * @param instanceName - The name of the instance that is being created. (not null) + * @param installConfig - The instance's install configuration. (not null) * @return The {@link RyaDetails} that were stored. - * @throws AlreadyInitializedException - * Could not be initialized because - * a table with this instance name has already exists and is holding the details. - * @throws RyaDetailsRepositoryException - * Something caused the initialization - * operation to fail. + * @throws AlreadyInitializedException Could not be initialized because a table with this instance name has already + * exists and is holding the details. + * @throws RyaDetailsRepositoryException Something caused the initialization operation to fail. */ - private RyaDetails initializeRyaDetails(final String instanceName, final InstallConfiguration installConfig) throws AlreadyInitializedException, RyaDetailsRepositoryException { + private RyaDetails initializeRyaDetails( + final String instanceName, + final InstallConfiguration installConfig) throws AlreadyInitializedException, RyaDetailsRepositoryException { final RyaDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getClient(), instanceName); // Build the PCJ Index details. [not supported in mongo] final PCJIndexDetails.Builder pcjDetailsBuilder = PCJIndexDetails.builder().setEnabled(false); final RyaDetails details = RyaDetails.builder() - // General Metadata - .setRyaInstanceName(instanceName).setRyaVersion(getVersion()) + // General Metadata + .setRyaInstanceName(instanceName).setRyaVersion(getVersion()) + + // FIXME RYA-215 .setGeoIndexDetails(new GeoIndexDetails(installConfig.isGeoIndexEnabled())) + + // Secondary Index Values + .setTemporalIndexDetails(new TemporalIndexDetails(installConfig.isTemporalIndexEnabled())) + .setFreeTextDetails(new FreeTextIndexDetails(installConfig.isFreeTextIndexEnabled()))// + + // Entity centric indexing is not supported in Mongo Db. + .setEntityCentricIndexDetails(new EntityCentricIndexDetails(false)) - // Secondary Index Values - // FIXME .setGeoIndexDetails(new GeoIndexDetails(installConfig.isGeoIndexEnabled())) - .setTemporalIndexDetails(new TemporalIndexDetails(installConfig.isTemporalIndexEnabled())) // - .setFreeTextDetails(new FreeTextIndexDetails(installConfig.isFreeTextIndexEnabled()))// - .setEntityCentricIndexDetails(new EntityCentricIndexDetails(false))// not supported in mongo - .setPCJIndexDetails(pcjDetailsBuilder) + .setPCJIndexDetails(pcjDetailsBuilder) - // Statistics values. - .setProspectorDetails(new ProspectorDetails(Optional. absent()))// - .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional. absent()))// - .build(); + // Statistics values. + .setProspectorDetails(new ProspectorDetails(Optional. absent()))// + .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional. absent()))// + .build(); // Initialize the table. detailsRepo.initialize(details); @@ -165,30 +166,34 @@ private RyaDetails initializeRyaDetails(final String instanceName, final Install * Builds a {@link MongoRdfConfiguration} object that will be used by the * Rya DAO to initialize all of the tables it will need. * - * @param connectionDetails - * - Indicates how to connect to Mongo. (not null) - * @param details - * - Indicates what needs to be installed. (not null) + * @param connectionDetails - Indicates how to connect to Mongo. (not null) + * @param ryaDetails - Indicates what needs to be installed. (not null) * @return A Rya Configuration object that can be used to perform the install. */ - private static MongoDBRdfConfiguration makeRyaConfig(final MongoConnectionDetails connectionDetails, final RyaDetails details) { - final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); - conf.setBoolean(ConfigUtils.USE_MONGO, true); - // The Rya Instance Name is used as a prefix for the index tables in Mongo. - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, details.getRyaInstanceName()); - - // Enable the indexers that the instance is configured to use. - conf.set(ConfigUtils.USE_PCJ, "" + details.getPCJIndexDetails().isEnabled()); - // fixme conf.set(OptionalConfigUtils.USE_GEO, "" + details.getGeoIndexDetails().isEnabled() ); - conf.set(ConfigUtils.USE_FREETEXT, "" + details.getFreeTextIndexDetails().isEnabled()); - conf.set(ConfigUtils.USE_TEMPORAL, "" + details.getTemporalIndexDetails().isEnabled()); + private static MongoDBRdfConfiguration makeRyaConfig(final MongoConnectionDetails connectionDetails, final RyaDetails ryaDetails) { + // Start with a configuration that is built using the connection details. + final MongoDBRdfConfiguration conf = connectionDetails.build(ryaDetails.getRyaInstanceName()); + + // The Mongo implementation of Rya does not currently support PCJs. + if(ryaDetails.getPCJIndexDetails().isEnabled()) { + log.warn("The install configuration says to enable PCJ indexing, but Mongo RYA does not support that " + + "feature. Ignoring this configuration."); + } + conf.set(ConfigUtils.USE_PCJ, "false"); // Mongo does not support entity indexing. - conf.set(ConfigUtils.USE_ENTITY, "" + false); + if(ryaDetails.getEntityCentricIndexDetails().isEnabled()) { + log.warn("The install configuration says to enable Entity Centric indexing, but Mongo RYA does not support " + + "that feature. Ignoring this configuration."); + } + conf.set(ConfigUtils.USE_ENTITY, "false"); + + // FIXME RYA-215 We haven't enabled geo indexing in the console yet. + //conf.set(OptionalConfigUtils.USE_GEO, "" + details.getGeoIndexDetails().isEnabled() ); - conf.set(ConfigUtils.CLOUDBASE_USER, connectionDetails.getUsername()); - conf.set(ConfigUtils.CLOUDBASE_PASSWORD, new String(connectionDetails.getPassword())); - conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, details.getRyaInstanceName()); + // Enable the supported indexers that the instance is configured to use. + conf.set(ConfigUtils.USE_FREETEXT, "" + ryaDetails.getFreeTextIndexDetails().isEnabled()); + conf.set(ConfigUtils.USE_TEMPORAL, "" + ryaDetails.getTemporalIndexDetails().isEnabled()); // This initializes the living indexers that will be used by the application and // caches them within the configuration object so that they may be used later. diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java index 256085657..a00079a79 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java @@ -21,7 +21,6 @@ import static java.util.Objects.requireNonNull; import org.apache.rya.api.client.InstanceExists; -import org.apache.rya.api.client.RyaClientException; import com.mongodb.MongoClient; @@ -38,8 +37,7 @@ public class MongoInstanceExists extends MongoCommand implements InstanceExists * Constructs an insatnce of {@link MongoInstanceExists}. * * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param client - Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ public MongoInstanceExists(final MongoConnectionDetails connectionDetails, final MongoClient client) { super(connectionDetails, client); @@ -48,6 +46,11 @@ public MongoInstanceExists(final MongoConnectionDetails connectionDetails, final @Override public boolean exists(final String instanceName) { requireNonNull( instanceName ); - return super.getClient().getDatabaseNames().contains(instanceName); + for(final String dbName : getClient().listDatabaseNames()) { + if(dbName.equals(instanceName)) { + return true; + } + } + return false; } -} +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java index d301bf2c1..bf2984875 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Set; import org.apache.rya.api.client.ListInstances; import org.apache.rya.api.client.RyaClientException; @@ -39,11 +38,8 @@ public class MongoListInstances extends MongoCommand implements ListInstances { /** * Constructs an instance of {@link MongoListInstances}. * - * @param connectionDetails - * - Details about the values that were used to create the connector to the cluster. (not null) - * @param client - * - Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) + * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ public MongoListInstances(final MongoConnectionDetails connectionDetails, final MongoClient client) { super(connectionDetails, client); @@ -52,12 +48,12 @@ public MongoListInstances(final MongoConnectionDetails connectionDetails, final @Override public List listInstances() throws RyaClientException { final MongoClient client = super.getClient(); - final List dbNames = client.getDatabaseNames(); + + // Each database that contains an instance details collection is a Rya Instance. final List ryaInstances = new ArrayList<>(); - for (final String db : dbNames) { - final Set collNames = client.getDB(db).getCollectionNames(); - for (final String coll : collNames) { - if (coll.equals(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME)) { + for (final String db : getClient().listDatabaseNames()) { + for(final String collection : client.getDatabase(db).listCollectionNames()) { + if (collection.equals(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME)) { ryaInstances.add(db); break; } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java index 4ea3487a9..f4d68e720 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java @@ -20,6 +20,8 @@ import static java.util.Objects.requireNonNull; +import java.util.Optional; + import org.apache.rya.api.client.RyaClient; import com.mongodb.MongoClient; @@ -38,29 +40,33 @@ public class MongoRyaClientFactory { * Initialize a set of {@link RyaClient} that will interact with an instance of * Rya that is hosted by a MongoDB server. * - * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param connector - The MongoDB connector the commands will use. (not null) + * @param connectionDetails - Details about the values that were used to connect to Mongo DB. (not null) + * @param mongoClient - The MongoDB client the commands will use. (not null) * @return The initialized commands. */ public static RyaClient build( final MongoConnectionDetails connectionDetails, - final MongoClient connector) { + final MongoClient mongoClient) { requireNonNull(connectionDetails); - requireNonNull(connector); + requireNonNull(mongoClient); // Build the RyaCommands option with the initialized commands. - return new RyaClient(// - new MongoInstall(connectionDetails, connector), // - new MongoCreatePCJ(connectionDetails, connector), // - new MongoDeletePCJ(connectionDetails, connector), // - null, null, null, null, - new MongoGetInstanceDetails(connectionDetails, connector), // - new MongoInstanceExists(connectionDetails, connector), // - new MongoListInstances(connectionDetails, connector), // - null, null, - new MongoUninstall(connectionDetails, connector), // - new MongoLoadStatements(connectionDetails, connector), // - new MongoLoadStatementsFile(connectionDetails, connector), // - null);// FIXME new MongoExecuteSparqlQuery(connectionDetails, connector)); + return new RyaClient( + new MongoInstall(connectionDetails, mongoClient), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + new MongoGetInstanceDetails(connectionDetails, mongoClient), + new MongoInstanceExists(connectionDetails, mongoClient), + new MongoListInstances(connectionDetails, mongoClient), + Optional.empty(), + Optional.empty(), + new MongoUninstall(connectionDetails, mongoClient), + new MongoLoadStatements(connectionDetails, mongoClient), + new MongoLoadStatementsFile(connectionDetails, mongoClient), + new MongoExecuteSparqlQuery(connectionDetails, mongoClient)); } } \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java index b3988b80e..979f93daf 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java @@ -19,6 +19,7 @@ package org.apache.rya.api.client.mongo; import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.client.Uninstall; @@ -34,23 +35,23 @@ @DefaultAnnotation(NonNull.class) public class MongoUninstall extends MongoCommand implements Uninstall { + private final InstanceExists instanceExists; + /** * Constructs an instance of {@link MongoUninstall}. * - * @param connectionDetails - * - Details about the values that were used to create the connector to the cluster. (not null) - * @param connector - * - Provides programatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) + * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ public MongoUninstall(final MongoConnectionDetails connectionDetails, final MongoClient client) { super(connectionDetails, client); + instanceExists = new MongoInstanceExists(connectionDetails, client); } @Override public void uninstall(final String ryaInstanceName) throws InstanceDoesNotExistException, RyaClientException { try { - if (!getClient().getDatabaseNames().contains(ryaInstanceName)) { + if (!instanceExists.exists(ryaInstanceName)) { throw new InstanceDoesNotExistException("The database '" + ryaInstanceName + "' does not exist."); } getClient().dropDatabase(ryaInstanceName); diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloAddUserIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloAddUserIT.java index 787d1104f..10d24d194 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloAddUserIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloAddUserIT.java @@ -46,7 +46,7 @@ */ public class AccumuloAddUserIT extends AccumuloITBase { - private String ADMIN_USER = testInstance.createUniqueUser(); + private final String ADMIN_USER = testInstance.createUniqueUser(); @Before public void beforeClass() throws Exception { @@ -92,7 +92,7 @@ public void ryaDetailsIncludesOriginalUser() throws Exception { */ @Test public void userAddedAlsoAddedToRyaDetails() throws Exception { - String user = testInstance.createUniqueUser(); + final String user = testInstance.createUniqueUser(); final SecurityOperations secOps = super.getConnector().securityOperations(); final RyaClient userAClient = AccumuloRyaClientFactory.build( @@ -106,7 +106,7 @@ public void userAddedAlsoAddedToRyaDetails() throws Exception { userAClient.getInstall().install(getRyaInstanceName(), InstallConfiguration.builder().build()); // Add the user. - userAClient.getAddUser().addUser(getRyaInstanceName(), user); + userAClient.getAddUser().get().addUser(getRyaInstanceName(), user); // Ensure the Rya instance's details have been updated to include the added user. final ImmutableList expectedUsers = ImmutableList.builder() @@ -123,7 +123,7 @@ public void userAddedAlsoAddedToRyaDetails() throws Exception { */ @Test public void userNotAddedCanNotInsert() throws Exception { - String user = testInstance.createUniqueUser(); + final String user = testInstance.createUniqueUser(); final SecurityOperations secOps = super.getConnector().securityOperations(); final RyaClient userAClient = AccumuloRyaClientFactory.build( @@ -171,7 +171,7 @@ public void userNotAddedCanNotInsert() throws Exception { */ @Test public void userAddedCanInsert() throws Exception { - String user = testInstance.createUniqueUser(); + final String user = testInstance.createUniqueUser(); final SecurityOperations secOps = super.getConnector().securityOperations(); final RyaClient userAClient = AccumuloRyaClientFactory.build( @@ -185,7 +185,7 @@ public void userAddedCanInsert() throws Exception { userAClient.getInstall().install(getRyaInstanceName(), InstallConfiguration.builder().build()); // Add the user. - userAClient.getAddUser().addUser(getRyaInstanceName(), user); + userAClient.getAddUser().get().addUser(getRyaInstanceName(), user); // Try to add a statement to the Rya instance. This should succeed. Sail sail = null; @@ -216,7 +216,7 @@ public void userAddedCanInsert() throws Exception { */ @Test public void addUserTwice() throws Exception { - String user = testInstance.createUniqueUser(); + final String user = testInstance.createUniqueUser(); final SecurityOperations secOps = super.getConnector().securityOperations(); final RyaClient userAClient = AccumuloRyaClientFactory.build( @@ -230,8 +230,8 @@ public void addUserTwice() throws Exception { userAClient.getInstall().install(getRyaInstanceName(), InstallConfiguration.builder().build()); // Add the user. - userAClient.getAddUser().addUser(getRyaInstanceName(), user); - userAClient.getAddUser().addUser(getRyaInstanceName(), user); + userAClient.getAddUser().get().addUser(getRyaInstanceName(), user); + userAClient.getAddUser().get().addUser(getRyaInstanceName(), user); // Ensure the Rya instance's details only contain the username of the user who installed the instance. final ImmutableList expectedUsers = ImmutableList.builder() diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloBatchUpdatePCJIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloBatchUpdatePCJIT.java index 5a2e69d06..40941c82a 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloBatchUpdatePCJIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloBatchUpdatePCJIT.java @@ -107,7 +107,7 @@ public void batchUpdate() throws Exception { final String pcjId = pcjStorage.createPcj(sparql); // Run the test. - ryaClient.getBatchUpdatePCJ().batchUpdate(RYA_INSTANCE_NAME, pcjId); + ryaClient.getBatchUpdatePCJ().get().batchUpdate(RYA_INSTANCE_NAME, pcjId); // Verify the correct results were loaded into the PCJ table. final Set expectedResults = new HashSet<>(); diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloRemoveUserIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloRemoveUserIT.java index f6ec5da51..224f70d34 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloRemoveUserIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/accumulo/AccumuloRemoveUserIT.java @@ -49,14 +49,14 @@ public class AccumuloRemoveUserIT extends AccumuloITBase { */ @Test public void removedUserNotInDetails() throws Exception { - String adminUser = testInstance.createUniqueUser(); - String user = testInstance.createUniqueUser(); + final String adminUser = testInstance.createUniqueUser(); + final String user = testInstance.createUniqueUser(); final SecurityOperations secOps = super.getConnector().securityOperations(); // Create the user that will install the instance of Rya. secOps.createLocalUser(adminUser, new PasswordToken(adminUser)); secOps.grantSystemPermission(adminUser, SystemPermission.CREATE_TABLE); - + final RyaClient userAClient = AccumuloRyaClientFactory.build( new AccumuloConnectionDetails(adminUser, adminUser.toCharArray(), getInstanceName(), getZookeepers()), @@ -73,10 +73,10 @@ public void removedUserNotInDetails() throws Exception { userAClient.getInstall().install(getRyaInstanceName(), InstallConfiguration.builder().build()); // Add userB. - userAClient.getAddUser().addUser(getRyaInstanceName(), user); + userAClient.getAddUser().get().addUser(getRyaInstanceName(), user); // Remove userA. - userBClient.getRemoveUser().removeUser(getRyaInstanceName(), adminUser); + userBClient.getRemoveUser().get().removeUser(getRyaInstanceName(), adminUser); // Ensure the Rya instance's details have been updated to include the added user. final ImmutableList expectedUsers = ImmutableList.builder() @@ -92,8 +92,8 @@ public void removedUserNotInDetails() throws Exception { */ @Test public void removedUserCanNotInsert() throws Exception { - String adminUser = testInstance.createUniqueUser(); - String user = testInstance.createUniqueUser(); + final String adminUser = testInstance.createUniqueUser(); + final String user = testInstance.createUniqueUser(); final SecurityOperations secOps = super.getConnector().securityOperations(); // Create the user that will install the instance of Rya. @@ -115,10 +115,10 @@ public void removedUserCanNotInsert() throws Exception { userAClient.getInstall().install(getRyaInstanceName(), InstallConfiguration.builder().build()); // Add userC. - userAClient.getAddUser().addUser(getRyaInstanceName(), user); + userAClient.getAddUser().get().addUser(getRyaInstanceName(), user); // Remove userA. - userCClient.getRemoveUser().removeUser(getRyaInstanceName(), adminUser); + userCClient.getRemoveUser().get().removeUser(getRyaInstanceName(), adminUser); // Show that userA can not insert anything. boolean securityExceptionThrown = false; diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java new file mode 100644 index 000000000..787db2b48 --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +/** + * Integration tests the methods of {@link }. + */ +public class MongoExecuteSparqlQueryIT { + +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java index 843ba42d4..4c70edaba 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java @@ -40,7 +40,6 @@ import org.junit.Test; import com.google.common.base.Optional; -import com.mongodb.BasicDBObject; import com.mongodb.MongoException; /** @@ -57,8 +56,7 @@ public void getDetails() throws MongoException, RyaClientException { .setEnableEntityCentricIndex(true) .setEnableFreeTextIndex(true) .setEnableTemporalIndex(true) - .setEnableGeoIndex(true) - .setEnablePcjIndex(false) + .setEnablePcjIndex(true) .build(); final Install install = new MongoInstall(getConnectionDetails(), this.getMongoClient()); @@ -74,13 +72,18 @@ public void getDetails() throws MongoException, RyaClientException { // The version depends on how the test is packaged, so just grab whatever was stored. .setRyaVersion( details.get().getRyaVersion() ) - // FIXME .setGeoIndexDetails( new GeoIndexDetails(true) ) + // The supported indices are set to true. .setTemporalIndexDetails(new TemporalIndexDetails(true) ) .setFreeTextDetails( new FreeTextIndexDetails(true) ) + + // Entity Centric Index is not supported, so it flips to false. .setEntityCentricIndexDetails( new EntityCentricIndexDetails(false) ) + + // PCJJ Index is not supported, so it flips to false. .setPCJIndexDetails( PCJIndexDetails.builder() .setEnabled(false)) + .setProspectorDetails( new ProspectorDetails(Optional.absent()) ) .setJoinSelectivityDetails( new JoinSelectivityDetails(Optional.absent()) ) .build(); @@ -99,25 +102,22 @@ public void getDetails_instanceDoesNotHaveDetails() throws MongoException, Table // Mimic a pre-details rya install. final String instanceName = "instance_name"; - this.getMongoClient().getDB(instanceName).createCollection("rya_triples", new BasicDBObject()); + this.getMongoClient().getDatabase(instanceName).createCollection("rya_triples"); // Verify that the operation returns empty. final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), this.getMongoClient()); final Optional details = getInstanceDetails.getDetails(instanceName); assertFalse( details.isPresent() ); } - + /** * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( - conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoDBName(), // aka instance - conf.getMongoInstance(), // aka hostname - conf.getCollectionName() - ); - return connectionDetails; + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt( conf.getMongoPort() )); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java index e9dae1c85..1ec7e9e7b 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -29,58 +30,53 @@ import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.RyaClientException; -import org.apache.rya.api.client.accumulo.AccumuloInstall; import org.apache.rya.mongodb.MongoTestBase; import org.junit.Test; /** - * Integration tests the methods of {@link AccumuloInstall}. + * Integration tests the methods of {@link MongoInstall}. */ public class MongoInstallIT extends MongoTestBase { @Test public void install() throws DuplicateInstanceNameException, RyaClientException { // Install an instance of Rya. - final String instanceName = conf.getCollectionName(); - final InstallConfiguration installConfig = InstallConfiguration.builder() - .setEnableTableHashPrefix(false) // - .setEnableEntityCentricIndex(false)// - .setEnableFreeTextIndex(false)// - .setEnableTemporalIndex(false)// - .setEnablePcjIndex(false)// - .setEnableGeoIndex(false)// - .setFluoPcjAppName("fluo_app_name")// - .build(); + final String ryaInstance = conf.getCollectionName(); + // Setup the connection details that were used for the embedded Mongo DB instance we are testing with. final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( - conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoDBName(), // aka instance - conf.getMongoInstance(), // aka hostname - conf.getCollectionName() - ); + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt( conf.getMongoPort() )); // Check that the instance does not exist. - assertFalse("Instance should NOT exist yet.", this.getMongoClient().getDatabaseNames().contains(instanceName)); final InstanceExists instanceExists = new MongoInstanceExists(connectionDetails, this.getMongoClient()); - assertFalse("Instance should NOT exist yet.", instanceExists.exists(instanceName)); + assertFalse(instanceExists.exists(ryaInstance)); + + // Install an instance of Rya with all the valid options turned on. + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(true) + .setEnableFreeTextIndex(true) + .setEnableTemporalIndex(true) + .build(); final Install install = new MongoInstall(connectionDetails, this.getMongoClient()); - install.install(instanceName, installConfig); + install.install(ryaInstance, installConfig); // Check that the instance exists. - assertTrue("Instance should exist.", this.getMongoClient().getDatabaseNames().contains(instanceName)); - List expected = Arrays.asList("instance_details", instanceName + "_triples"); + assertTrue(instanceExists.exists(ryaInstance)); + + // Show that the expected collections were created within the database. + final List expected = Arrays.asList("instance_details", ryaInstance + "_triples"); int count = 0; - String found = ""; - for (String collection : this.getMongoClient().getDatabase(instanceName).listCollectionNames()) - { - System.out.println("Collection names:" + collection); + final List found = new ArrayList<>(); + for (final String collection : this.getMongoClient().getDatabase(ryaInstance).listCollectionNames()) { count += expected.contains(collection) ? 1 : 0; - found += ", " + collection; + found.add( collection ); } assertTrue("Tables missing from:" + expected + " actual:" + found, expected.size() == count); - assertTrue("Instance should exist.", instanceExists.exists(instanceName)); + assertTrue("Instance should exist.", instanceExists.exists(ryaInstance)); } @Test(expected = DuplicateInstanceNameException.class) @@ -89,12 +85,11 @@ public void install_alreadyExists() throws DuplicateInstanceNameException, RyaCl final String instanceName = conf.getCollectionName(); final InstallConfiguration installConfig = InstallConfiguration.builder().build(); - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoDBName(), // aka instance - conf.getMongoInstance(), // aka hostname - conf.getCollectionName() - ); + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt( conf.getMongoPort() )); final Install install = new MongoInstall(connectionDetails, this.getMongoClient()); install.install(instanceName, installConfig); diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java index 35568bcf9..6611f7f55 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java @@ -22,17 +22,15 @@ import static org.junit.Assert.assertTrue; import org.apache.accumulo.core.client.TableExistsException; -import org.apache.rya.api.client.accumulo.AccumuloInstanceExists; import org.apache.rya.mongodb.MongoTestBase; import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; import org.junit.Test; -import com.mongodb.BasicDBObject; import com.mongodb.MongoClient; import com.mongodb.MongoException; /** - * Integration tests the methods of {@link AccumuloInstanceExists}. + * Integration tests the methods of {@link MongoInstanceExistsExists}. */ public class MongoInstanceExistsIT extends MongoTestBase { @@ -42,7 +40,7 @@ public void exists_ryaDetailsTable() throws MongoException, TableExistsException // Create the Rya instance's Rya details collection. final String instanceName = "test_instance_"; - client.getDB(instanceName).createCollection(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME, new BasicDBObject()); + client.getDatabase(instanceName).createCollection(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME); // Verify the command reports the instance exists. final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), client); @@ -55,7 +53,7 @@ public void exists_dataTables() throws MongoException, TableExistsException { // Create the Rya instance's Rya triples collection. final String instanceName = "test_instance_"; - client.getDB(instanceName).createCollection("rya_triples", new BasicDBObject()); + client.getDatabase(instanceName).createCollection("rya_triples"); // Verify the command reports the instance exists. final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), client); @@ -73,14 +71,10 @@ public void doesNotExist() throws MongoException { * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoDBName(), // aka instance - conf.getMongoInstance(), // aka hostname - conf.getCollectionName() - ); - return connectionDetails; + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt( conf.getMongoPort() )); } - - } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java index 7d5a2fe74..0e9f37efc 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java @@ -60,12 +60,10 @@ public void listInstances_hasRyaDetailsTable() throws MongoException, DuplicateI * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoDBName(), // aka instance - conf.getMongoInstance(), // aka hostname - conf.getCollectionName() - ); - return connectionDetails; + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt( conf.getMongoPort() )); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index 6794d865c..500bf31ad 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -18,115 +18,97 @@ */ package org.apache.rya.api.client.mongo; -import static org.junit.Assert.assertEquals; - -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - -import org.apache.rya.api.client.Install; -import org.apache.rya.api.client.Install.InstallConfiguration; -import org.apache.rya.api.client.InstanceDoesNotExistException; -import org.apache.rya.api.client.RyaClient; import org.apache.rya.mongodb.MongoTestBase; -import org.bson.Document; -import org.junit.Test; -import org.openrdf.model.Statement; -import org.openrdf.model.ValueFactory; -import org.openrdf.model.impl.ValueFactoryImpl; -import org.openrdf.rio.RDFFormat; - -import com.mongodb.client.MongoCursor; /** * Integration tests the methods of {@link MongoLoadStatementsFile}. */ public class MongoLoadStatementsFileIT extends MongoTestBase { - @Test(expected = InstanceDoesNotExistException.class) - public void instanceDoesNotExist() throws Exception { - - final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); - ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getInstance(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); - } - - @Test - public void loadTurtleFile() throws Exception { - // Install an instance of Rya. - final InstallConfiguration installConfig = InstallConfiguration.builder() - .setEnableTableHashPrefix(false) - .setEnableEntityCentricIndex(false) - .setEnableFreeTextIndex(false) - .setEnableTemporalIndex(false) - .setEnablePcjIndex(false) - .setEnableGeoIndex(false) - .build(); - MongoConnectionDetails connectionDetails = getConnectionDetails(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); - final Install install = ryaClient.getInstall(); - install.install(connectionDetails.getInstance(), installConfig); - - // Load the test statement file. - ryaClient.getLoadStatementsFile().loadStatements( // - connectionDetails.getInstance(), // - Paths.get("src/test/resources/example.ttl"), // - RDFFormat.TURTLE); - - // Verify that the statements were loaded. - final ValueFactory vf = new ValueFactoryImpl(); - - final List expected = new ArrayList<>(); - expected.add( vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob")) ); - expected.add( vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie")) ); - expected.add( vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream")) ); - - final List statements = new ArrayList<>(); - MongoCursor x = getRyaCollection().find().iterator(); - System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); - while (x.hasNext()) { - Document y = x.next(); - System.out.println("getRyaCollection()=" + y); - } - assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); - // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); - // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); - // final Iterator> it = scanner.iterator(); - // while(it.hasNext()) { - // final Entry next = it.next(); - // - // final Key key = next.getKey(); - // final byte[] row = key.getRow().getBytes(); - // final byte[] columnFamily = key.getColumnFamily().getBytes(); - // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); - // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); - // - // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); - // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); - // - // // Filter out the rya version statement if it is present. - // if(!isRyaMetadataStatement(vf, statement)) { - // statements.add( statement ); - // } - // } - // - // assertEquals(expected, statements); - } - - private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { - return statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#version") ) || - statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#rts") ); - } - /** - * @return copy from conf to MongoConnectionDetails - */ - private MongoConnectionDetails getConnectionDetails() { - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(// - conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoDBName(), // aka instance - conf.getMongoInstance(), // aka hostname - conf.getCollectionName() - ); - return connectionDetails; - } +// @Test(expected = InstanceDoesNotExistException.class) +// public void instanceDoesNotExist() throws Exception { +// +// final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); +// ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getInstance(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); +// } +// +// @Test +// public void loadTurtleFile() throws Exception { +// // Install an instance of Rya. +// final InstallConfiguration installConfig = InstallConfiguration.builder() +// .setEnableTableHashPrefix(false) +// .setEnableEntityCentricIndex(false) +// .setEnableFreeTextIndex(false) +// .setEnableTemporalIndex(false) +// .setEnablePcjIndex(false) +// .setEnableGeoIndex(false) +// .build(); +// MongoConnectionDetails connectionDetails = getConnectionDetails(); +// final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); +// final Install install = ryaClient.getInstall(); +// install.install(connectionDetails.getInstance(), installConfig); +// +// // Load the test statement file. +// ryaClient.getLoadStatementsFile().loadStatements( // +// connectionDetails.getInstance(), // +// Paths.get("src/test/resources/example.ttl"), // +// RDFFormat.TURTLE); +// +// // Verify that the statements were loaded. +// final ValueFactory vf = new ValueFactoryImpl(); +// +// final List expected = new ArrayList<>(); +// expected.add( vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob")) ); +// expected.add( vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie")) ); +// expected.add( vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream")) ); +// +// final List statements = new ArrayList<>(); +// MongoCursor x = getRyaCollection().find().iterator(); +// System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); +// while (x.hasNext()) { +// Document y = x.next(); +// System.out.println("getRyaCollection()=" + y); +// } +// assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); +// // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); +// // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); +// // final Iterator> it = scanner.iterator(); +// // while(it.hasNext()) { +// // final Entry next = it.next(); +// // +// // final Key key = next.getKey(); +// // final byte[] row = key.getRow().getBytes(); +// // final byte[] columnFamily = key.getColumnFamily().getBytes(); +// // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); +// // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); +// // +// // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); +// // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); +// // +// // // Filter out the rya version statement if it is present. +// // if(!isRyaMetadataStatement(vf, statement)) { +// // statements.add( statement ); +// // } +// // } +// // +// // assertEquals(expected, statements); +// } +// +// private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { +// return statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#version") ) || +// statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#rts") ); +// } +// /** +// * @return copy from conf to MongoConnectionDetails +// */ +// private MongoConnectionDetails getConnectionDetails() { +// final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(// +// conf.getMongoUser(), // +// conf.getMongoPassword().toCharArray(), // +// conf.getMongoDBName(), // aka instance +// conf.getMongoInstance(), // aka hostname +// conf.getCollectionName() +// ); +// return connectionDetails; +// } } diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java new file mode 100644 index 000000000..fb616bccb --- /dev/null +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java @@ -0,0 +1,26 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.api.client.mongo; + +/** + * Integration tests the methods of {@link MongoLoadStatements}. + */ +public class MongoLoadStatementsIT { + +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java index 523b4b260..dffb0f2e5 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java @@ -18,6 +18,9 @@ */ package org.apache.rya.api.client.mongo; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + import org.apache.rya.api.client.Install; import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceDoesNotExistException; @@ -25,13 +28,12 @@ import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.client.Uninstall; import org.apache.rya.mongodb.MongoTestBase; -import org.junit.Assert; import org.junit.Test; import com.mongodb.MongoException; /** - * Integration tests the methods of {@link MongoInstall}. + * Integration tests the methods of {@link MongoUninstall}. */ public class MongoUninstallIT extends MongoTestBase { @@ -44,13 +46,16 @@ public void uninstall() throws MongoException, RyaClientException { final Install install = new MongoInstall(getConnectionDetails(), conf.getMongoClient()); install.install(instanceName, installConfig); + // Show that the instance exists. + final InstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), conf.getMongoClient()); + assertTrue( instanceExists.exists(instanceName) ); + // Uninstall the instance final Uninstall uninstall = new MongoUninstall(getConnectionDetails(), conf.getMongoClient()); uninstall.uninstall(instanceName); // Check that the instance no longer exists. - final InstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), conf.getMongoClient()); - Assert.assertFalse(instanceExists.exists(instanceName)); + assertFalse(instanceExists.exists(instanceName)); } @Test(expected = InstanceDoesNotExistException.class) @@ -67,12 +72,10 @@ public void uninstall_instanceDoesNotExists() throws MongoException, RyaClientEx * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoDBName(), // aka instance - conf.getMongoInstance(), // aka hostname - conf.getCollectionName() - ); - return connectionDetails; + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt( conf.getMongoPort() )); } } \ No newline at end of file diff --git a/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/periodic/KafkaLatencyBenchmark.java b/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/periodic/KafkaLatencyBenchmark.java index 454f7e0dc..28a5e5ddf 100644 --- a/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/periodic/KafkaLatencyBenchmark.java +++ b/extras/rya.benchmark/src/main/java/org/apache/rya/benchmark/periodic/KafkaLatencyBenchmark.java @@ -172,7 +172,7 @@ private String issueQuery() throws InstanceDoesNotExistException, RyaClientExcep + "group by ?type"; logger.info("Query: {}", sparql); - return client.getCreatePCJ().createPCJ(options.getRyaInstance(), sparql, ImmutableSet.of(ExportStrategy.KAFKA)); + return client.getCreatePCJ().get().createPCJ(options.getRyaInstance(), sparql, ImmutableSet.of(ExportStrategy.KAFKA)); } private String issuePeriodicQuery(final PeriodicQueryCommand periodicOptions) throws InstanceDoesNotExistException, RyaClientException { @@ -184,7 +184,7 @@ private String issuePeriodicQuery(final PeriodicQueryCommand periodicOptions) th + "?obs ?type } " + "group by ?type"; logger.info("Query: {}", sparql); - final String queryId = client.getCreatePeriodicPCJ().createPeriodicPCJ(options.getRyaInstance(), sparql, periodicOptions.getPeriodicQueryRegistrationTopic(), options.getKafkaBootstrap()); + final String queryId = client.getCreatePeriodicPCJ().get().createPeriodicPCJ(options.getRyaInstance(), sparql, periodicOptions.getPeriodicQueryRegistrationTopic(), options.getKafkaBootstrap()); logger.info("Received query id: {}", queryId); return queryId.substring("QUERY_".length()); // remove the QUERY_ prefix. } diff --git a/extras/rya.benchmark/src/test/java/org/apache/rya/benchmark/query/QueryBenchmarkRunIT.java b/extras/rya.benchmark/src/test/java/org/apache/rya/benchmark/query/QueryBenchmarkRunIT.java index dd5fe68a1..cc5ba8ba8 100644 --- a/extras/rya.benchmark/src/test/java/org/apache/rya/benchmark/query/QueryBenchmarkRunIT.java +++ b/extras/rya.benchmark/src/test/java/org/apache/rya/benchmark/query/QueryBenchmarkRunIT.java @@ -148,7 +148,7 @@ private static void createTestPCJ(final RyaClient ryaClient) throws Exception { final String pcjId = pcjs.createPcj(SPARQL_QUERY); // Batch update the PCJ using the Rya Client. - ryaClient.getBatchUpdatePCJ().batchUpdate(RYA_INSTANCE_NAME, pcjId); + ryaClient.getBatchUpdatePCJ().get().batchUpdate(RYA_INSTANCE_NAME, pcjId); } } diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java index ef5ab342c..7d6b24103 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/CreateDeleteIT.java @@ -132,7 +132,7 @@ private String loadData(final String sparql, final Collection stateme // Register the PCJ with Rya. final RyaClient ryaClient = AccumuloRyaClientFactory.build(createConnectionDetails(), getAccumuloConnector()); - final String pcjId = ryaClient.getCreatePCJ().createPCJ(getRyaInstanceName(), sparql, Sets.newHashSet()); + final String pcjId = ryaClient.getCreatePCJ().get().createPCJ(getRyaInstanceName(), sparql, Sets.newHashSet()); // Write the data to Rya. final SailRepositoryConnection ryaConn = getRyaSailRepository().getConnection(); diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java index 9c557aacb..3e72f1b62 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/integration/QueryIT.java @@ -430,25 +430,25 @@ public void withTemporal() throws Exception { public void dateTimeWithin() throws Exception { final ValueFactory vf = new ValueFactoryImpl(); - DatatypeFactory dtf = DatatypeFactory.newInstance(); + final DatatypeFactory dtf = DatatypeFactory.newInstance(); FunctionRegistry.getInstance().add(new DateTimeWithinPeriod()); final String sparql = "PREFIX fn: <" + FN.NAMESPACE +">" + "SELECT ?event ?startTime ?endTime WHERE { ?event ?startTime; ?endTime. " + "FILTER(fn:dateTimeWithin(?startTime, ?endTime, 2,<" + OWLTime.HOURS_URI + "> ))}"; - ZonedDateTime zTime = ZonedDateTime.now(); - String time = zTime.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime = ZonedDateTime.now(); + final String time = zTime.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime1 = zTime.minusHours(1); - String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime1 = zTime.minusHours(1); + final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime2 = zTime.minusHours(2); - String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime2 = zTime.minusHours(2); + final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); - Literal lit = vf.createLiteral(dtf.newXMLGregorianCalendar(time)); - Literal lit1 = vf.createLiteral(dtf.newXMLGregorianCalendar(time1)); - Literal lit2 = vf.createLiteral(dtf.newXMLGregorianCalendar(time2)); + final Literal lit = vf.createLiteral(dtf.newXMLGregorianCalendar(time)); + final Literal lit1 = vf.createLiteral(dtf.newXMLGregorianCalendar(time1)); + final Literal lit2 = vf.createLiteral(dtf.newXMLGregorianCalendar(time2)); // Create the Statements that will be loaded into Rya. final Collection statements = Sets.newHashSet( @@ -461,7 +461,7 @@ public void dateTimeWithin() throws Exception { // Create the expected results of the SPARQL query once the PCJ has been computed. final Set expectedResults = new HashSet<>(); - MapBindingSet bs = new MapBindingSet(); + final MapBindingSet bs = new MapBindingSet(); bs.addBinding("event", vf.createURI("uri:event1")); bs.addBinding("startTime", lit); bs.addBinding("endTime", lit1); @@ -475,21 +475,21 @@ public void dateTimeWithin() throws Exception { public void dateTimeWithinNow() throws Exception { final ValueFactory vf = new ValueFactoryImpl(); - DatatypeFactory dtf = DatatypeFactory.newInstance(); + final DatatypeFactory dtf = DatatypeFactory.newInstance(); FunctionRegistry.getInstance().add(new DateTimeWithinPeriod()); final String sparql = "PREFIX fn: <" + FN.NAMESPACE +">" + "SELECT ?event ?startTime WHERE { ?event ?startTime. " + "FILTER(fn:dateTimeWithin(?startTime, NOW(), 30, <" + OWLTime.SECONDS_URI + "> ))}"; - ZonedDateTime zTime = ZonedDateTime.now(); - String time = zTime.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime = ZonedDateTime.now(); + final String time = zTime.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime1 = zTime.minusSeconds(30); - String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime1 = zTime.minusSeconds(30); + final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); - Literal lit = vf.createLiteral(dtf.newXMLGregorianCalendar(time)); - Literal lit1 = vf.createLiteral(dtf.newXMLGregorianCalendar(time1)); + final Literal lit = vf.createLiteral(dtf.newXMLGregorianCalendar(time)); + final Literal lit1 = vf.createLiteral(dtf.newXMLGregorianCalendar(time1)); // Create the Statements that will be loaded into Rya. final Collection statements = Sets.newHashSet( @@ -500,7 +500,7 @@ public void dateTimeWithinNow() throws Exception { // Create the expected results of the SPARQL query once the PCJ has been computed. final Set expectedResults = new HashSet<>(); - MapBindingSet bs = new MapBindingSet(); + final MapBindingSet bs = new MapBindingSet(); bs.addBinding("event", vf.createURI("uri:event1")); bs.addBinding("startTime", lit); expectedResults.add(bs); @@ -513,7 +513,7 @@ public void dateTimeWithinNow() throws Exception { @Test public void periodicQueryTestWithoutAggregation() throws Exception { - String query = "prefix function: " // n + final String query = "prefix function: " // n + "prefix time: " // n + "select ?id where {" // n + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n @@ -523,20 +523,20 @@ public void periodicQueryTestWithoutAggregation() throws Exception { // Create the Statements that will be loaded into Rya. final ValueFactory vf = new ValueFactoryImpl(); final DatatypeFactory dtf = DatatypeFactory.newInstance(); - ZonedDateTime time = ZonedDateTime.now(); - long currentTime = time.toInstant().toEpochMilli(); + final ZonedDateTime time = ZonedDateTime.now(); + final long currentTime = time.toInstant().toEpochMilli(); - ZonedDateTime zTime1 = time.minusMinutes(30); - String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime1 = time.minusMinutes(30); + final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime2 = zTime1.minusMinutes(30); - String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime2 = zTime1.minusMinutes(30); + final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime3 = zTime2.minusMinutes(30); - String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime3 = zTime2.minusMinutes(30); + final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime4 = zTime3.minusMinutes(30); - String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime4 = zTime3.minusMinutes(30); + final String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); final Collection statements = Sets.newHashSet( vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), @@ -555,8 +555,8 @@ public void periodicQueryTestWithoutAggregation() throws Exception { // Create the expected results of the SPARQL query once the PCJ has been computed. final Set expectedResults = new HashSet<>(); - long period = 1800000; - long binId = (currentTime / period) * period; + final long period = 1800000; + final long binId = (currentTime / period) * period; MapBindingSet bs = new MapBindingSet(); bs.addBinding("id", vf.createLiteral("id_1", XMLSchema.STRING)); @@ -614,7 +614,7 @@ public void periodicQueryTestWithoutAggregation() throws Exception { @Test public void periodicQueryTestWithAggregation() throws Exception { - String query = "prefix function: " // n + final String query = "prefix function: " // n + "prefix time: " // n + "select (count(?obs) as ?total) where {" // n + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n @@ -624,20 +624,20 @@ public void periodicQueryTestWithAggregation() throws Exception { // Create the Statements that will be loaded into Rya. final ValueFactory vf = new ValueFactoryImpl(); final DatatypeFactory dtf = DatatypeFactory.newInstance(); - ZonedDateTime time = ZonedDateTime.now(); - long currentTime = time.toInstant().toEpochMilli(); + final ZonedDateTime time = ZonedDateTime.now(); + final long currentTime = time.toInstant().toEpochMilli(); - ZonedDateTime zTime1 = time.minusMinutes(30); - String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime1 = time.minusMinutes(30); + final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime2 = zTime1.minusMinutes(30); - String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime2 = zTime1.minusMinutes(30); + final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime3 = zTime2.minusMinutes(30); - String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime3 = zTime2.minusMinutes(30); + final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime4 = zTime3.minusMinutes(30); - String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime4 = zTime3.minusMinutes(30); + final String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); final Collection statements = Sets.newHashSet( vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), @@ -656,8 +656,8 @@ public void periodicQueryTestWithAggregation() throws Exception { // Create the expected results of the SPARQL query once the PCJ has been computed. final Set expectedResults = new HashSet<>(); - long period = 1800000; - long binId = (currentTime / period) * period; + final long period = 1800000; + final long binId = (currentTime / period) * period; MapBindingSet bs = new MapBindingSet(); bs.addBinding("total", vf.createLiteral("4", XMLSchema.INTEGER)); @@ -685,7 +685,7 @@ public void periodicQueryTestWithAggregation() throws Exception { @Test public void periodicQueryTestWithAggregationAndGroupBy() throws Exception { - String query = "prefix function: " // n + final String query = "prefix function: " // n + "prefix time: " // n + "select ?id (count(?obs) as ?total) where {" // n + "Filter(function:periodic(?time, 2, .5, time:hours)) " // n @@ -695,20 +695,20 @@ public void periodicQueryTestWithAggregationAndGroupBy() throws Exception { // Create the Statements that will be loaded into Rya. final ValueFactory vf = new ValueFactoryImpl(); final DatatypeFactory dtf = DatatypeFactory.newInstance(); - ZonedDateTime time = ZonedDateTime.now(); - long currentTime = time.toInstant().toEpochMilli(); + final ZonedDateTime time = ZonedDateTime.now(); + final long currentTime = time.toInstant().toEpochMilli(); - ZonedDateTime zTime1 = time.minusMinutes(30); - String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime1 = time.minusMinutes(30); + final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime2 = zTime1.minusMinutes(30); - String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime2 = zTime1.minusMinutes(30); + final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime3 = zTime2.minusMinutes(30); - String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime3 = zTime2.minusMinutes(30); + final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime4 = zTime3.minusMinutes(30); - String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime4 = zTime3.minusMinutes(30); + final String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); final Collection statements = Sets.newHashSet( vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), @@ -733,8 +733,8 @@ public void periodicQueryTestWithAggregationAndGroupBy() throws Exception { // Create the expected results of the SPARQL query once the PCJ has been computed. final Set expectedResults = new HashSet<>(); - long period = 1800000; - long binId = (currentTime / period) * period; + final long period = 1800000; + final long binId = (currentTime / period) * period; MapBindingSet bs = new MapBindingSet(); bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER)); @@ -803,7 +803,7 @@ public void periodicQueryTestWithAggregationAndGroupBy() throws Exception { @Test public void nestedPeriodicQueryTestWithAggregationAndGroupBy() throws Exception { - String query = "prefix function: " // n + final String query = "prefix function: " // n + "prefix time: " // n + "select ?location ?total " + "where { Filter(?total > 1) {" @@ -815,20 +815,20 @@ public void nestedPeriodicQueryTestWithAggregationAndGroupBy() throws Exception // Create the Statements that will be loaded into Rya. final ValueFactory vf = new ValueFactoryImpl(); final DatatypeFactory dtf = DatatypeFactory.newInstance(); - ZonedDateTime time = ZonedDateTime.now(); - long currentTime = time.toInstant().toEpochMilli(); + final ZonedDateTime time = ZonedDateTime.now(); + final long currentTime = time.toInstant().toEpochMilli(); - ZonedDateTime zTime1 = time.minusMinutes(30); - String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime1 = time.minusMinutes(30); + final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime2 = zTime1.minusMinutes(30); - String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime2 = zTime1.minusMinutes(30); + final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime3 = zTime2.minusMinutes(30); - String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime3 = zTime2.minusMinutes(30); + final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime4 = zTime3.minusMinutes(30); - String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime4 = zTime3.minusMinutes(30); + final String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); final Collection statements = Sets.newHashSet( vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), @@ -853,8 +853,8 @@ public void nestedPeriodicQueryTestWithAggregationAndGroupBy() throws Exception // Create the expected results of the SPARQL query once the PCJ has been computed. final Set expectedResults = new HashSet<>(); - long period = 1800000; - long binId = (currentTime / period) * period; + final long period = 1800000; + final long binId = (currentTime / period) * period; MapBindingSet bs = new MapBindingSet(); bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER)); @@ -881,7 +881,7 @@ public void nestedPeriodicQueryTestWithAggregationAndGroupBy() throws Exception @Test public void nestedJoinPeriodicQueryWithAggregationAndGroupBy() throws Exception { - String query = "prefix function: " // n + final String query = "prefix function: " // n + "prefix time: " // n + "select ?location ?total ?population " + "where { Filter(?total > 1)" @@ -894,20 +894,20 @@ public void nestedJoinPeriodicQueryWithAggregationAndGroupBy() throws Exception // Create the Statements that will be loaded into Rya. final ValueFactory vf = new ValueFactoryImpl(); final DatatypeFactory dtf = DatatypeFactory.newInstance(); - ZonedDateTime time = ZonedDateTime.now(); - long currentTime = time.toInstant().toEpochMilli(); + final ZonedDateTime time = ZonedDateTime.now(); + final long currentTime = time.toInstant().toEpochMilli(); - ZonedDateTime zTime1 = time.minusMinutes(30); - String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime1 = time.minusMinutes(30); + final String time1 = zTime1.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime2 = zTime1.minusMinutes(30); - String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime2 = zTime1.minusMinutes(30); + final String time2 = zTime2.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime3 = zTime2.minusMinutes(30); - String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime3 = zTime2.minusMinutes(30); + final String time3 = zTime3.format(DateTimeFormatter.ISO_INSTANT); - ZonedDateTime zTime4 = zTime3.minusMinutes(30); - String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); + final ZonedDateTime zTime4 = zTime3.minusMinutes(30); + final String time4 = zTime4.format(DateTimeFormatter.ISO_INSTANT); final Collection statements = Sets.newHashSet( vf.createStatement(vf.createURI("urn:obs_1"), vf.createURI("uri:hasTime"), @@ -934,8 +934,8 @@ public void nestedJoinPeriodicQueryWithAggregationAndGroupBy() throws Exception // Create the expected results of the SPARQL query once the PCJ has been computed. final Set expectedResults = new HashSet<>(); - long period = 1800000; - long binId = (currentTime / period) * period; + final long period = 1800000; + final long binId = (currentTime / period) * period; MapBindingSet bs = new MapBindingSet(); bs.addBinding("total", vf.createLiteral("2", XMLSchema.INTEGER)); @@ -965,7 +965,7 @@ public void nestedJoinPeriodicQueryWithAggregationAndGroupBy() throws Exception @Test(expected= UnsupportedQueryException.class) public void nestedConstructPeriodicQueryWithAggregationAndGroupBy() throws Exception { - String query = "prefix function: " // n + final String query = "prefix function: " // n + "prefix time: " // n + "construct{?location a } " + "where { Filter(?total > 1)" @@ -984,7 +984,7 @@ public void nestedConstructPeriodicQueryWithAggregationAndGroupBy() throws Excep } public void runTest(final String sparql, final Collection statements, final Collection expectedResults, - ExportStrategy strategy) throws Exception { + final ExportStrategy strategy) throws Exception { requireNonNull(sparql); requireNonNull(statements); requireNonNull(expectedResults); @@ -996,7 +996,7 @@ public void runTest(final String sparql, final Collection statements, switch (strategy) { case RYA: - ryaClient.getCreatePCJ().createPCJ(getRyaInstanceName(), sparql); + ryaClient.getCreatePCJ().get().createPCJ(getRyaInstanceName(), sparql); addStatementsAndWait(statements); // Fetch the value that is stored within the PCJ table. try (final PrecomputedJoinStorage pcjStorage = new AccumuloPcjStorage(accumuloConn, getRyaInstanceName())) { @@ -1008,8 +1008,8 @@ public void runTest(final String sparql, final Collection statements, break; case PERIODIC: - PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(accumuloConn, getRyaInstanceName()); - String periodicId = periodicStorage.createPeriodicQuery(sparql); + final PeriodicQueryResultStorage periodicStorage = new AccumuloPeriodicQueryResultStorage(accumuloConn, getRyaInstanceName()); + final String periodicId = periodicStorage.createPeriodicQuery(sparql); try (FluoClient fluo = new FluoClientImpl(super.getFluoConfiguration())) { new CreateFluoPcj().createPcj(periodicId, sparql, Sets.newHashSet(ExportStrategy.PERIODIC), fluo); } diff --git a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java index 27900d466..8529bd5c4 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java +++ b/extras/rya.pcj.fluo/pcj.fluo.integration/src/test/java/org/apache/rya/indexing/pcj/fluo/visibility/PcjVisibilityIT.java @@ -104,7 +104,7 @@ public void visibilitySimplified() throws Exception { final RyaClient ryaClient = AccumuloRyaClientFactory.build(createConnectionDetails(), accumuloConn); - final String pcjId = ryaClient.getCreatePCJ().createPCJ(getRyaInstanceName(), sparql); + final String pcjId = ryaClient.getCreatePCJ().get().createPCJ(getRyaInstanceName(), sparql); // Grant the root user the "u" authorization. super.getAccumuloConnector().securityOperations().changeUserAuthorizations(getUsername(), new Authorizations("u")); diff --git a/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java index ca17b2adf..7314ce538 100644 --- a/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java +++ b/extras/rya.pcj.fluo/pcj.fluo.test.base/src/main/java/org/apache/rya/pcj/fluo/test/base/KafkaExportITBase.java @@ -185,7 +185,7 @@ public void teardownRya() { // Shutdown the repo. if(ryaSailRepo != null) {ryaSailRepo.shutDown();} if(dao != null ) {dao.destroy();} - } catch (Exception e) { + } catch (final Exception e) { System.out.println("Encountered the following Exception when shutting down Rya: " + e.getMessage()); } } @@ -342,7 +342,7 @@ protected String loadDataAndCreateQuery(final String sparql, final Collection statements, accInstance.getInstanceName(), accInstance.getZooKeepers()), accumuloConn); - ryaClient.getCreatePCJ().createPCJ(getRyaInstanceName(), sparql); + ryaClient.getCreatePCJ().get().createPCJ(getRyaInstanceName(), sparql); // Write the data to Rya. final SailRepositoryConnection ryaConn = super.getRyaSailRepository().getConnection(); diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java index 335880623..8b86d43fc 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java @@ -291,9 +291,9 @@ public String printInstanceDetails() { @CliCommand(value = CREATE_PCJ_CMD, help = "Creates and starts the maintenance of a new PCJ using a Fluo application.") public String createPcj( - @CliOption(key = {"exportToRya"}, mandatory = false, help = "Indicates that results for the query should be exported to a Rya PCJ table.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") + @CliOption(key = {"exportToRya"}, mandatory = false, help = "Indicates that results for the query should be exported to a Rya PCJ table.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") final boolean exportToRya, - @CliOption(key = {"exportToKafka"}, mandatory = false, help = "Indicates that results for the query should be exported to a Kafka Topic.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") + @CliOption(key = {"exportToKafka"}, mandatory = false, help = "Indicates that results for the query should be exported to a Kafka Topic.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") final boolean exportToKafka) { // Fetch the command that is connected to the store. final ShellState shellState = state.getShellState(); @@ -311,12 +311,12 @@ public String createPcj( if(strategies.isEmpty()) { return "The user must specify at least one export strategy: (--exportToRya, --exportToKafka)"; } - + // Prompt the user for the SPARQL. final Optional sparql = sparqlPrompt.getSparql(); if (sparql.isPresent()) { // Execute the command. - final String pcjId = commands.getCreatePCJ().createPCJ(ryaInstance, sparql.get(), strategies); + final String pcjId = commands.getCreatePCJ().get().createPCJ(ryaInstance, sparql.get(), strategies); // Return a message that indicates the ID of the newly created ID. return String.format("The PCJ has been created. Its ID is '%s'.", pcjId); } else { @@ -340,7 +340,7 @@ public String deletePcj( try { // Execute the command. - commands.getDeletePCJ().deletePCJ(ryaInstance, pcjId); + commands.getDeletePCJ().get().deletePCJ(ryaInstance, pcjId); return "The PCJ has been deleted."; } catch (final InstanceDoesNotExistException e) { @@ -349,12 +349,12 @@ public String deletePcj( throw new RuntimeException("The PCJ could not be deleted. Provided reason: " + e.getMessage(), e); } } - + @CliCommand(value = CREATE_PERIODIC_PCJ_CMD, help = "Creates and starts the maintenance of a new Periodic PCJ and registers the associated Periodic Notification with Kafka.") public String createPeriodicPcj( - @CliOption(key = {"topic"}, mandatory = true, help = "Kafka topic for registering new PeriodicNotifications. This topic is monitored by the Periodic Notification Service.") + @CliOption(key = {"topic"}, mandatory = true, help = "Kafka topic for registering new PeriodicNotifications. This topic is monitored by the Periodic Notification Service.") final String topic, - @CliOption(key = {"brokers"}, mandatory = true, help = "Comma delimited list of host/port pairs to establish the initial connection to the Kafka cluster.") + @CliOption(key = {"brokers"}, mandatory = true, help = "Comma delimited list of host/port pairs to establish the initial connection to the Kafka cluster.") final String brokers) { // Fetch the command that is connected to the store. final ShellState shellState = state.getShellState(); @@ -366,7 +366,7 @@ public String createPeriodicPcj( final Optional sparql = sparqlPrompt.getSparql(); if (sparql.isPresent()) { // Execute the command. - final String pcjId = commands.getCreatePeriodicPCJ().createPeriodicPCJ(ryaInstance, sparql.get(), topic, brokers); + final String pcjId = commands.getCreatePeriodicPCJ().get().createPeriodicPCJ(ryaInstance, sparql.get(), topic, brokers); // Return a message that indicates the ID of the newly created ID. return String.format("The Periodic PCJ has been created. Its ID is '%s'.", pcjId); } else { @@ -378,7 +378,7 @@ public String createPeriodicPcj( throw new RuntimeException("Could not create the Periodic PCJ. Provided reasons: " + e.getMessage(), e); } } - + @CliCommand(value = DELETE_PERIODIC_PCJ_CMD, help = "Deletes and halts maintenance of a Periodic PCJ.") public String deletePeriodicPcj( @CliOption(key = {"pcjId"}, mandatory = true, help = "The ID of the PCJ that will be deleted.") @@ -395,7 +395,7 @@ public String deletePeriodicPcj( try { // Execute the command. - commands.getDeletePeriodicPCJ().deletePeriodicPCJ(ryaInstance, pcjId, topic, brokers); + commands.getDeletePeriodicPCJ().get().deletePeriodicPCJ(ryaInstance, pcjId, topic, brokers); return "The Periodic PCJ has been deleted."; } catch (final InstanceDoesNotExistException e) { @@ -404,8 +404,8 @@ public String deletePeriodicPcj( throw new RuntimeException("The Periodic PCJ could not be deleted. Provided reason: " + e.getMessage(), e); } } - - + + @CliCommand(value = LIST_INCREMENTAL_QUERIES, help = "Lists relevant information about all SPARQL queries maintained by the Fluo application.") public String listFluoQueries() { // Fetch the command that is connected to the store. @@ -414,14 +414,14 @@ public String listFluoQueries() { final String ryaInstance = shellState.getRyaInstanceName().get(); try { - return commands.getListIncrementalQueries().listIncrementalQueries(ryaInstance); + return commands.getListIncrementalQueries().get().listIncrementalQueries(ryaInstance); } catch (final InstanceDoesNotExistException e) { throw new RuntimeException(String.format("A Rya instance named '%s' does not exist.", ryaInstance), e); - } catch (RyaClientException e) { + } catch (final RyaClientException e) { throw new RuntimeException("Could not list incremental queries. Provided reasons: " + e.getMessage(), e); } } - + @CliCommand(value = ADD_USER_CMD, help = "Adds an authorized user to the Rya instance.") public void addUser( @@ -433,7 +433,7 @@ public void addUser( final String ryaInstance = shellState.getRyaInstanceName().get(); try { - ryaClient.getAddUser().addUser(ryaInstance, username); + ryaClient.getAddUser().get().addUser(ryaInstance, username); } catch (final InstanceDoesNotExistException e) { throw new RuntimeException(String.format("A Rya instance named '%s' does not exist.", ryaInstance), e); } catch (final RyaClientException e) { @@ -451,7 +451,7 @@ public void removeUser( final String ryaInstance = shellState.getRyaInstanceName().get(); try { - ryaClient.getRemoveUser().removeUser(ryaInstance, username); + ryaClient.getRemoveUser().get().removeUser(ryaInstance, username); } catch (final InstanceDoesNotExistException e) { throw new RuntimeException(String.format("A Rya instance named '%s' does not exist.", ryaInstance), e); } catch (final RyaClientException e) { diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java index 1249536dc..6e21f8dbd 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java @@ -33,6 +33,8 @@ import org.apache.rya.api.client.AddUser; import org.apache.rya.api.client.CreatePCJ; +import org.apache.rya.api.client.CreatePCJ.ExportStrategy; +import org.apache.rya.api.client.CreatePeriodicPCJ; import org.apache.rya.api.client.DeletePCJ; import org.apache.rya.api.client.DeletePeriodicPCJ; import org.apache.rya.api.client.GetInstanceDetails; @@ -45,8 +47,6 @@ import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.client.Uninstall; -import org.apache.rya.api.client.CreatePCJ.ExportStrategy; -import org.apache.rya.api.client.CreatePeriodicPCJ; import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; import org.apache.rya.api.instance.RyaDetails; import org.apache.rya.api.instance.RyaDetails.EntityCentricIndexDetails; @@ -83,7 +83,7 @@ public void createPCJ() throws InstanceDoesNotExistException, RyaClientException when(mockCreatePCJ.createPCJ( eq(instanceName), eq(sparql), eq(strategies) ) ).thenReturn( pcjId ); final RyaClient mockCommands = mock(RyaClient.class); - when(mockCommands.getCreatePCJ()).thenReturn( mockCreatePCJ ); + when(mockCommands.getCreatePCJ()).thenReturn( java.util.Optional.of(mockCreatePCJ) ); final SharedShellState state = new SharedShellState(); state.connectedToAccumulo(mock(AccumuloConnectionDetails.class), mockCommands); @@ -152,7 +152,7 @@ public void deletePCJ() throws InstanceDoesNotExistException, RyaClientException final DeletePCJ mockDeletePCJ = mock(DeletePCJ.class); final RyaClient mockCommands = mock(RyaClient.class); - when(mockCommands.getDeletePCJ()).thenReturn( mockDeletePCJ ); + when(mockCommands.getDeletePCJ()).thenReturn( java.util.Optional.of(mockDeletePCJ) ); final SharedShellState state = new SharedShellState(); state.connectedToAccumulo(mock(AccumuloConnectionDetails.class), mockCommands); @@ -172,7 +172,7 @@ public void deletePCJ() throws InstanceDoesNotExistException, RyaClientException final String expected = "The PCJ has been deleted."; assertEquals(expected, message); } - + @Test public void createPeriodicPCJ() throws InstanceDoesNotExistException, RyaClientException, IOException { // Mock the object that performs the create operation. @@ -185,7 +185,7 @@ public void createPeriodicPCJ() throws InstanceDoesNotExistException, RyaClientE when(mockCreatePCJ.createPeriodicPCJ( eq(instanceName), eq(sparql), eq(topic), eq(brokers) )).thenReturn( pcjId ); final RyaClient mockCommands = mock(RyaClient.class); - when(mockCommands.getCreatePeriodicPCJ()).thenReturn( mockCreatePCJ ); + when(mockCommands.getCreatePeriodicPCJ()).thenReturn( java.util.Optional.of(mockCreatePCJ) ); final SharedShellState state = new SharedShellState(); state.connectedToAccumulo(mock(AccumuloConnectionDetails.class), mockCommands); @@ -205,14 +205,14 @@ public void createPeriodicPCJ() throws InstanceDoesNotExistException, RyaClientE final String expected = "The Periodic PCJ has been created. Its ID is '12341234'."; assertEquals(expected, message); } - + @Test public void deletePeriodicPCJ() throws InstanceDoesNotExistException, RyaClientException { // Mock the object that performs the delete operation. final DeletePeriodicPCJ mockDeletePCJ = mock(DeletePeriodicPCJ.class); final RyaClient mockCommands = mock(RyaClient.class); - when(mockCommands.getDeletePeriodicPCJ()).thenReturn( mockDeletePCJ ); + when(mockCommands.getDeletePeriodicPCJ()).thenReturn( java.util.Optional.of(mockDeletePCJ) ); final SharedShellState state = new SharedShellState(); state.connectedToAccumulo(mock(AccumuloConnectionDetails.class), mockCommands); @@ -234,7 +234,7 @@ public void deletePeriodicPCJ() throws InstanceDoesNotExistException, RyaClientE final String expected = "The Periodic PCJ has been deleted."; assertEquals(expected, message); } - + @Test public void getInstanceDetails() throws InstanceDoesNotExistException, RyaClientException { @@ -478,7 +478,7 @@ public void addUser() throws Exception { final AddUser mockAddUser = mock(AddUser.class); final RyaClient mockClient = mock(RyaClient.class); - when(mockClient.getAddUser()).thenReturn( mockAddUser ); + when(mockClient.getAddUser()).thenReturn( java.util.Optional.of(mockAddUser) ); final SharedShellState state = new SharedShellState(); state.connectedToAccumulo(mock(AccumuloConnectionDetails.class), mockClient); @@ -498,7 +498,7 @@ public void removeUser() throws Exception { final RemoveUser mockRemoveUser = mock(RemoveUser.class); final RyaClient mockClient = mock(RyaClient.class); - when(mockClient.getRemoveUser()).thenReturn( mockRemoveUser ); + when(mockClient.getRemoveUser()).thenReturn( java.util.Optional.of(mockRemoveUser) ); final SharedShellState state = new SharedShellState(); state.connectedToAccumulo(mock(AccumuloConnectionDetails.class), mockClient); From 82d421b29c1ce20cb793f5c12a6601f71c5f01ce Mon Sep 17 00:00:00 2001 From: "David W. Lotts" Date: Fri, 15 Dec 2017 17:19:45 -0500 Subject: [PATCH 03/13] RYA-104 LoadStatementFile is not reading for some reason, todo: MongoExecSparql. --- .../rya/mongodb/EmbeddedMongoFactory.java | 2 +- .../client/mongo/MongoConnectionDetails.java | 10 +- .../client/mongo/MongoLoadStatementsFile.java | 6 +- .../mongo/MongoExecuteSparqlQueryIT.java | 26 ++- .../mongo/MongoLoadStatementsFileIT.java | 189 ++++++++++-------- 5 files changed, 141 insertions(+), 92 deletions(-) diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java index b3cfaa4ee..a2c25ed6b 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/EmbeddedMongoFactory.java @@ -101,4 +101,4 @@ public void shutdown() { mongodProcess.stop(); mongodExecutable.stop(); } -} \ No newline at end of file +} diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index f05b38d03..81106f94e 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -24,6 +24,8 @@ import org.apache.rya.indexing.accumulo.ConfigUtils; import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import com.mongodb.MongoClient; + import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -92,6 +94,10 @@ public int getPort() { * @return Constructs a new {@link MongoDBRdfConfiguration} object with values from this object. */ public MongoDBRdfConfiguration build(final String ryaInstanceName) { + return build(ryaInstanceName, null); + } + + public MongoDBRdfConfiguration build(final String ryaInstanceName, MongoClient mongoClient) { // Note, we don't use the MongoDBRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the MongoRdfConfiguration object, it may do the right thing. @@ -106,7 +112,9 @@ public MongoDBRdfConfiguration build(final String ryaInstanceName) { // Both of these are ways to configure the collection prefixes. conf.setCollectionName(ryaInstanceName); conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, ryaInstanceName); - + if (mongoClient != null) { + conf.setMongoClient(mongoClient); + } return conf; } } \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java index bf525c1f0..6229a3f01 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java @@ -67,12 +67,12 @@ public void loadStatements(String ryaInstanceName, Path statementsFile, RDFForma SailRepository sailRepo = null; SailRepositoryConnection sailRepoConn = null; // Get a Sail object that is connected to the Rya instance. - final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName); + final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName, getClient()); // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. try { sail = RyaSailFactory.getInstance(ryaConf); - } catch (SailException | AccumuloException | AccumuloSecurityException | RyaDAOException | InferenceEngineException e) { - throw new RyaClientException("While getting an sail instance.", e); + } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException | AccumuloSecurityException e) { + throw new RyaClientException("While getting a sail instance.", e); } // Load the file. diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java index 787db2b48..153992c4c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java @@ -18,9 +18,33 @@ */ package org.apache.rya.api.client.mongo; +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.DuplicateInstanceNameException; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.mongodb.MongoTestBase; +import org.junit.Test; + +import com.mongodb.MongoException; + /** * Integration tests the methods of {@link }. */ -public class MongoExecuteSparqlQueryIT { +public class MongoExecuteSparqlQueryIT extends MongoTestBase { + @Test + public void ExecuteSparqlQuery_exec() throws MongoException, DuplicateInstanceNameException, RyaClientException { + MongoConnectionDetails connectionDetails = getConnectionDetails(); + // Install a few instances of Rya using the install command. + final Install install = new MongoInstall(connectionDetails, getMongoClient()); + install.install("instanceExec", InstallConfiguration.builder().build()); + MongoExecuteSparqlQuery executeSparql = new MongoExecuteSparqlQuery(connectionDetails, getMongoClient()); + // TODO executeSparql. + } + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + return new MongoConnectionDetails(conf.getMongoUser(), conf.getMongoPassword().toCharArray(), conf.getMongoInstance(), Integer.parseInt(conf.getMongoPort())); + } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index 500bf31ad..c3b7cd0a1 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -18,97 +18,114 @@ */ package org.apache.rya.api.client.mongo; +import static org.junit.Assert.assertEquals; + +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClient; import org.apache.rya.mongodb.MongoTestBase; +import org.bson.Document; +import org.junit.Test; +import org.openrdf.model.Statement; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.ValueFactoryImpl; +import org.openrdf.rio.RDFFormat; +import com.mongodb.client.MongoCursor; /** * Integration tests the methods of {@link MongoLoadStatementsFile}. */ public class MongoLoadStatementsFileIT extends MongoTestBase { -// @Test(expected = InstanceDoesNotExistException.class) -// public void instanceDoesNotExist() throws Exception { -// -// final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); -// ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getInstance(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); -// } -// -// @Test -// public void loadTurtleFile() throws Exception { -// // Install an instance of Rya. -// final InstallConfiguration installConfig = InstallConfiguration.builder() -// .setEnableTableHashPrefix(false) -// .setEnableEntityCentricIndex(false) -// .setEnableFreeTextIndex(false) -// .setEnableTemporalIndex(false) -// .setEnablePcjIndex(false) -// .setEnableGeoIndex(false) -// .build(); -// MongoConnectionDetails connectionDetails = getConnectionDetails(); -// final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); -// final Install install = ryaClient.getInstall(); -// install.install(connectionDetails.getInstance(), installConfig); -// -// // Load the test statement file. -// ryaClient.getLoadStatementsFile().loadStatements( // -// connectionDetails.getInstance(), // -// Paths.get("src/test/resources/example.ttl"), // -// RDFFormat.TURTLE); -// -// // Verify that the statements were loaded. -// final ValueFactory vf = new ValueFactoryImpl(); -// -// final List expected = new ArrayList<>(); -// expected.add( vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob")) ); -// expected.add( vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie")) ); -// expected.add( vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream")) ); -// -// final List statements = new ArrayList<>(); -// MongoCursor x = getRyaCollection().find().iterator(); -// System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); -// while (x.hasNext()) { -// Document y = x.next(); -// System.out.println("getRyaCollection()=" + y); -// } -// assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); -// // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); -// // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); -// // final Iterator> it = scanner.iterator(); -// // while(it.hasNext()) { -// // final Entry next = it.next(); -// // -// // final Key key = next.getKey(); -// // final byte[] row = key.getRow().getBytes(); -// // final byte[] columnFamily = key.getColumnFamily().getBytes(); -// // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); -// // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); -// // -// // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); -// // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); -// // -// // // Filter out the rya version statement if it is present. -// // if(!isRyaMetadataStatement(vf, statement)) { -// // statements.add( statement ); -// // } -// // } -// // -// // assertEquals(expected, statements); -// } -// -// private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { -// return statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#version") ) || -// statement.getPredicate().equals( vf.createURI("urn:org.apache.rya/2012/05#rts") ); -// } -// /** -// * @return copy from conf to MongoConnectionDetails -// */ -// private MongoConnectionDetails getConnectionDetails() { -// final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(// -// conf.getMongoUser(), // -// conf.getMongoPassword().toCharArray(), // -// conf.getMongoDBName(), // aka instance -// conf.getMongoInstance(), // aka hostname -// conf.getCollectionName() -// ); -// return connectionDetails; -// } + @Test(expected = InstanceDoesNotExistException.class) + public void instanceDoesNotExist() throws Exception { + + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); + ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getHostname(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); + } + + @Test + public void loadTurtleFile() throws Exception { + // Install an instance of Rya. + final InstallConfiguration installConfig = InstallConfiguration.builder()// + .setEnableTableHashPrefix(false)// + .setEnableEntityCentricIndex(false)// + .setEnableFreeTextIndex(false)// + .setEnableTemporalIndex(false)// + .setEnablePcjIndex(false)// + .setEnableGeoIndex(false)// + .build(); + MongoConnectionDetails connectionDetails = getConnectionDetails(); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final Install install = ryaClient.getInstall(); + install.install(conf.getCollectionName(), installConfig); + + // Load the test statement file. + ryaClient.getLoadStatementsFile() + // LoadStatementsFile loadStatementsFile = new MongoLoadStatementsFile(connectionDetails, getMongoClient()); + // loadStatementsFile + .loadStatements( // + conf.getCollectionName(), // + Paths.get("src/test/resources/example.ttl"), // + RDFFormat.TURTLE); + + // Verify that the statements were loaded. + final ValueFactory vf = new ValueFactoryImpl(); + + final List expected = new ArrayList<>(); + expected.add(vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob"))); + expected.add(vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie"))); + expected.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream"))); + + final List statements = new ArrayList<>(); + MongoCursor x = getRyaCollection().find().iterator(); + System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); + while (x.hasNext()) { + Document y = x.next(); + System.out.println("getRyaCollection()=" + y); + } + assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); + // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); + // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); + // final Iterator> it = scanner.iterator(); + // while(it.hasNext()) { + // final Entry next = it.next(); + // + // final Key key = next.getKey(); + // final byte[] row = key.getRow().getBytes(); + // final byte[] columnFamily = key.getColumnFamily().getBytes(); + // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); + // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); + // + // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); + // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); + // + // // Filter out the rya version statement if it is present. + // if(!isRyaMetadataStatement(vf, statement)) { + // statements.add( statement ); + // } + // } + // + // assertEquals(expected, statements); + } + + private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { + return statement.getPredicate().equals(vf.createURI("urn:org.apache.rya/2012/05#version")) || statement.getPredicate().equals(vf.createURI("urn:org.apache.rya/2012/05#rts")); + } + + /** + * @return copy from conf to MongoConnectionDetails + */ + private MongoConnectionDetails getConnectionDetails() { + return new MongoConnectionDetails(// + conf.getMongoUser(), // + conf.getMongoPassword().toCharArray(), // + conf.getMongoInstance(), // + Integer.parseInt(conf.getMongoPort())); + } } From b6422618a40dc90418e4488ab0ed134a18d160bf Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Sun, 17 Dec 2017 14:56:03 -0500 Subject: [PATCH 04/13] RYA-104 Making the build pass. --- .../api/client/mongo/MongoLoadStatements.java | 18 ++++++++++++++++++ .../client/mongo/MongoLoadStatementsFile.java | 18 ++++++++++++++++++ .../api/client/mongo/MongoListInstancesIT.java | 2 +- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java index 844489da7..846adfe5c 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.rya.api.client.mongo; import org.apache.rya.api.client.InstanceDoesNotExistException; diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java index 6229a3f01..9f689444c 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java @@ -1,3 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package org.apache.rya.api.client.mongo; import static java.util.Objects.requireNonNull; diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java index 0e9f37efc..561434d1e 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java @@ -31,7 +31,7 @@ import org.apache.rya.mongodb.MongoTestBase; import org.junit.Test; -import com.beust.jcommander.internal.Lists; +import com.google.common.collect.Lists; import com.mongodb.MongoException; /** From 9e8f4a654f47edc7d06fa5ba7bc7011f4716549c Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Sun, 17 Dec 2017 15:23:41 -0500 Subject: [PATCH 05/13] RYA-104 Implemented the Rya Shell integration with the Mongo DB interactors. --- .../client/mongo/MongoConnectionDetails.java | 4 +- .../apache/rya/shell/RyaAdminCommands.java | 112 +++++++++++--- .../rya/shell/RyaConnectionCommands.java | 84 +++++++++-- .../apache/rya/shell/RyaPromptProvider.java | 7 +- .../apache/rya/shell/SharedShellState.java | 138 +++++++++++++++--- .../rya/shell/RyaAdminCommandsTest.java | 50 ++++++- .../rya/shell/SharedShellStateTest.java | 9 +- 7 files changed, 337 insertions(+), 67 deletions(-) diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index 81106f94e..d7922892a 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -46,7 +46,7 @@ public class MongoConnectionDetails { * @param username - The username that was used to establish the connection. (not null) * @param password - The password that was used to establish the connection. (not null) * @param hostname - The hostname of the Mongo DB that was connected to. (not null) - * @param port - The port of the Mongo DB that was connected to. (not null) + * @param port - The port of the Mongo DB that was connected to. */ public MongoConnectionDetails( final String username, @@ -97,7 +97,7 @@ public MongoDBRdfConfiguration build(final String ryaInstanceName) { return build(ryaInstanceName, null); } - public MongoDBRdfConfiguration build(final String ryaInstanceName, MongoClient mongoClient) { + public MongoDBRdfConfiguration build(final String ryaInstanceName, final MongoClient mongoClient) { // Note, we don't use the MongoDBRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the MongoRdfConfiguration object, it may do the right thing. diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java index 8b86d43fc..cfe13dd6e 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java @@ -36,6 +36,7 @@ import org.apache.rya.api.instance.RyaDetails; import org.apache.rya.shell.SharedShellState.ConnectionState; import org.apache.rya.shell.SharedShellState.ShellState; +import org.apache.rya.shell.SharedShellState.StorageType; import org.apache.rya.shell.util.InstallPrompt; import org.apache.rya.shell.util.InstanceNamesFormatter; import org.apache.rya.shell.util.RyaDetailsFormatter; @@ -63,7 +64,8 @@ public class RyaAdminCommands implements CommandMarker { public static final String LIST_INCREMENTAL_QUERIES = "list-incremental-queries"; public static final String PRINT_INSTANCE_DETAILS_CMD = "print-instance-details"; public static final String INSTALL_CMD = "install"; - public static final String INSTALL_PARAMETERS_CMD = "install-with-parameters"; + public static final String INSTALL_ACCUMULO_PARAMETERS_CMD = "install-with-accumulo-parameters"; + public static final String INSTALL_MONGO_PARAMETERS_CMD = "install-with-mongo-parameters"; public static final String LIST_INSTANCES_CMD = "list-instances"; public static final String UNINSTALL_CMD = "uninstall"; public static final String ADD_USER_CMD = "add-user"; @@ -99,8 +101,7 @@ public RyaAdminCommands( */ @CliAvailabilityIndicator({ LIST_INSTANCES_CMD, - INSTALL_CMD, - INSTALL_PARAMETERS_CMD}) + INSTALL_CMD}) public boolean areStorageCommandsAvailable() { switch(state.getShellState().getConnectionState()) { case CONNECTED_TO_STORAGE: @@ -111,21 +112,53 @@ public boolean areStorageCommandsAvailable() { } } + /** + * Enables commands that only become available once the Shell has been connected to an Accumulo Rya Storage. + */ + @CliAvailabilityIndicator({ + INSTALL_ACCUMULO_PARAMETERS_CMD}) + public boolean areAccumuloStorageCommandsAvailable() { + return isConnectedToStorageType(StorageType.ACCUMULO); + } + + /** + * Enables commands that only become available once the Shell has been connected to an MongoDB Rya Storage. + */ + @CliAvailabilityIndicator({ + INSTALL_MONGO_PARAMETERS_CMD}) + public boolean areMongoStorageCommandsAvailable() { + return isConnectedToStorageType(StorageType.MONGO); + } + + private boolean isConnectedToStorageType(final StorageType type) { + // Enabled if we are connected to the specified storage type. + final Optional storageType = state.getShellState().getStorageType(); + if(storageType.isPresent()) { + return areStorageCommandsAvailable() && storageType.get() == type; + } + + // Otherwise disabled. + return false; + } + /** * Enables commands that are always available once the Shell is connected to a Rya Instance. */ @CliAvailabilityIndicator({ PRINT_INSTANCE_DETAILS_CMD, - UNINSTALL_CMD, + UNINSTALL_CMD}) + public boolean areInstanceCommandsAvailable() { + return state.getShellState().getConnectionState() == ConnectionState.CONNECTED_TO_INSTANCE; + } + + /** + * Enables commands that are available when connected to a Rya Instance that supports user management. + */ + @CliAvailabilityIndicator({ ADD_USER_CMD, REMOVE_USER_CMD}) - public boolean areInstanceCommandsAvailable() { - switch(state.getShellState().getConnectionState()) { - case CONNECTED_TO_INSTANCE: - return true; - default: - return false; - } + public boolean areUserCommandAvailable() { + return areInstanceCommandsAvailable() && state.getShellState().getStorageType().get() == StorageType.ACCUMULO; } /** @@ -141,7 +174,8 @@ public boolean arePCJCommandsAvailable() { // The PCJ commands are only available if the Shell is connected to an instance of Rya // that is new enough to use the RyaDetailsRepository and is configured to maintain PCJs. final ShellState shellState = state.getShellState(); - if(shellState.getConnectionState() == ConnectionState.CONNECTED_TO_INSTANCE) { + if(shellState.getConnectionState() == ConnectionState.CONNECTED_TO_INSTANCE && + shellState.getStorageType().get() == StorageType.ACCUMULO) { final GetInstanceDetails getInstanceDetails = shellState.getConnectedCommands().get().getGetInstanceDetails(); final String ryaInstanceName = state.getShellState().getRyaInstanceName().get(); try { @@ -211,8 +245,8 @@ public String install() { } } - @CliCommand(value = INSTALL_PARAMETERS_CMD, help = "Create a new instance of Rya with command line parameters.") - public String installWithParameters( + @CliCommand(value = INSTALL_ACCUMULO_PARAMETERS_CMD, help = "Create a new Accumulo instance of Rya with command line parameters.") + public String installWithAccumuloParameters( @CliOption(key = {"instanceName"}, mandatory = true, help = "The name of the Rya instance to create.") final String instanceName, @@ -225,8 +259,9 @@ public String installWithParameters( @CliOption(key = {"enableFreeTextIndex"}, mandatory = false, help = "Use Free Text Indexing.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") final boolean enableFreeTextIndex, - @CliOption(key = {"enableGeospatialIndex"}, mandatory = false, help = "Use Geospatial Indexing.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") - final boolean enableGeospatialIndex, + // TODO RYA-215 +// @CliOption(key = {"enableGeospatialIndex"}, mandatory = false, help = "Use Geospatial Indexing.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") +// final boolean enableGeospatialIndex, @CliOption(key = {"enableTemporalIndex"}, mandatory = false, help = "Use Temporal Indexing.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") final boolean enableTemporalIndex, @@ -246,7 +281,8 @@ public String installWithParameters( .setEnableTableHashPrefix(enableTableHashPrefix) .setEnableEntityCentricIndex(enableEntityCentricIndex) .setEnableFreeTextIndex(enableFreeTextIndex) - .setEnableGeoIndex(enableGeospatialIndex) + // TODO RYA-215 +// .setEnableGeoIndex(enableGeospatialIndex) .setEnableTemporalIndex(enableTemporalIndex) .setEnablePcjIndex(enablePcjIndex) .setFluoPcjAppName(fluoPcjAppName) @@ -268,6 +304,48 @@ public String installWithParameters( } } + @CliCommand(value = INSTALL_MONGO_PARAMETERS_CMD, help = "Create a new MongoDB instance of Rya with command line parameters.") + public String installWithMongoParameters( + @CliOption(key = {"instanceName"}, mandatory = true, help = "The name of the Rya instance to create.") + final String instanceName, + + @CliOption(key = {"enableFreeTextIndex"}, mandatory = false, help = "Use Free Text Indexing.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") + final boolean enableFreeTextIndex, + + // TODO RYA-215 +// @CliOption(key = {"enableGeospatialIndex"}, mandatory = false, help = "Use Geospatial Indexing.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") +// final boolean enableGeospatialIndex, + + @CliOption(key = {"enableTemporalIndex"}, mandatory = false, help = "Use Temporal Indexing.", unspecifiedDefaultValue = "false", specifiedDefaultValue = "true") + final boolean enableTemporalIndex) { + + // Fetch the commands that are connected to the store. + final RyaClient commands = state.getShellState().getConnectedCommands().get(); + + try { + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableFreeTextIndex(enableFreeTextIndex) + // TODO RYA-215 +// .setEnableGeoIndex(enableGeospatialIndex) + .setEnableTemporalIndex(enableTemporalIndex) + .build(); + + // Verify the configuration is what the user actually wants to do. + if (!installPrompt.promptVerified(instanceName, installConfig)) { + return "Skipping Installation."; + } + + // Execute the command. + commands.getInstall().install(instanceName, installConfig); + return String.format("The Rya instance named '%s' has been installed.", instanceName); + + } catch(final DuplicateInstanceNameException e) { + throw new RuntimeException(String.format("A Rya instance named '%s' already exists. Try again with a different name.", instanceName), e); + } catch (final IOException | RyaClientException e) { + throw new RuntimeException("Could not install a new instance of Rya. Reason: " + e.getMessage(), e); + } + } + @CliCommand(value = PRINT_INSTANCE_DETAILS_CMD, help = "Print information about how the Rya instance is configured.") public String printInstanceDetails() { // Fetch the command that is connected to the store. diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java index f5ba45172..e51896859 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java @@ -31,7 +31,12 @@ import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; import org.apache.rya.api.client.accumulo.AccumuloRyaClientFactory; +import org.apache.rya.api.client.mongo.MongoConnectionDetails; +import org.apache.rya.api.client.mongo.MongoRyaClientFactory; +import org.apache.rya.mongodb.MongoConnectorFactory; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; import org.apache.rya.shell.SharedShellState.ConnectionState; +import org.apache.rya.shell.SharedShellState.StorageType; import org.apache.rya.shell.util.ConnectorFactory; import org.apache.rya.shell.util.PasswordPrompt; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +47,7 @@ import org.springframework.stereotype.Component; import com.google.common.base.Optional; +import com.mongodb.MongoClient; /** * Spring Shell commands that manage the connection that is used by the shell. @@ -52,6 +58,7 @@ public class RyaConnectionCommands implements CommandMarker { // Command line commands. public static final String PRINT_CONNECTION_DETAILS_CMD = "print-connection-details"; public static final String CONNECT_ACCUMULO_CMD = "connect-accumulo"; + public static final String CONNECT_MONGO_CMD = "connect-mongo"; public static final String CONNECT_INSTANCE_CMD = "connect-rya"; public static final String DISCONNECT_COMMAND_NAME_CMD = "disconnect"; @@ -75,7 +82,7 @@ public boolean isPrintConnectionDetailsAvailable() { return true; } - @CliAvailabilityIndicator({CONNECT_ACCUMULO_CMD}) + @CliAvailabilityIndicator({CONNECT_ACCUMULO_CMD, CONNECT_MONGO_CMD}) public boolean areConnectCommandsAvailable() { return sharedState.getShellState().getConnectionState() == ConnectionState.DISCONNECTED; } @@ -98,17 +105,31 @@ public boolean isDisconnectAvailable() { @CliCommand(value = PRINT_CONNECTION_DETAILS_CMD, help = "Print information about the Shell's Rya storage connection.") public String printConnectionDetails() { - final Optional detailsHolder = sharedState.getShellState().getConnectionDetails(); - - if(detailsHolder.isPresent()) { - final AccumuloConnectionDetails details = detailsHolder.get(); - return "The shell is connected to an instance of Accumulo using the following parameters:\n" + - " Username: " + details.getUsername() + "\n" + - " Instance Name: " + details.getInstanceName() + "\n" + - " Zookeepers: " + details.getZookeepers(); - } else { + // Check to see if the shell is connected to any storages. + final Optional storageType = sharedState.getShellState().getStorageType(); + if(!storageType.isPresent()) { return "The shell is not connected to anything."; } + + // Create a print out based on what it is connected to. + switch(storageType.get()) { + case ACCUMULO: + final AccumuloConnectionDetails accDetails = sharedState.getShellState().getAccumuloDetails().get(); + return "The shell is connected to an instance of Accumulo using the following parameters:\n" + + " Username: " + accDetails.getUsername() + "\n" + + " Instance Name: " + accDetails.getInstanceName() + "\n" + + " Zookeepers: " + accDetails.getZookeepers(); + + case MONGO: + final MongoConnectionDetails mongoDetails = sharedState.getShellState().getMongoDetails().get(); + return "The shell is connected to an instance of MongoDB using the following parameters:\n" + + " Hostname: " + mongoDetails.getHostname() + "\n" + + " Port: " + mongoDetails.getPort() + "\n" + + " Username:" + mongoDetails.getUsername(); + + default: + throw new RuntimeException("Unrecognized StorageType: " + storageType.get()); + } } @CliCommand(value = CONNECT_ACCUMULO_CMD, help = "Connect the shell to an instance of Accumulo.") @@ -138,6 +159,49 @@ public String connectToAccumulo( return "Connected. You must select a Rya instance to interact with next."; } + @CliCommand(value = CONNECT_MONGO_CMD, help = "Connect the shell to an instance of MongoDB.") + public String connectToMongo( + @CliOption(key = {"username"}, mandatory = true, help = "The username that will be used to connect to MongoDB.") + final String username, + @CliOption(key= {"hostname"}, mandatory = true, help = "The hostname of the MongoDB that will be connected to.") + final String hostname, + @CliOption(key= {"port"}, mandatory = true, help = "The port of the MongoDB that will be connected to.") + final String port) { + + // Prompt the user for their password. + try { + final char[] password = passwordPrompt.getPassword(); + + // Set up a configuration file that connects to the specified Mongo DB. + final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); + conf.setMongoInstance(hostname); + conf.setMongoPort(port); + conf.setMongoUser(username); + conf.setMongoPassword(new String(password)); + + // Create the singleton instance of Mongo that will be used through out the application. + final MongoClient client = MongoConnectorFactory.getMongoClient(conf); + + // Make sure the client is closed at shutdown. + Runtime.getRuntime().addShutdownHook(new Thread() { + @Override + public void run() { + MongoConnectorFactory.closeMongoClient(); + } + }); + + // Initialize the connected to Mongo shared state. + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(username, password, hostname, Integer.parseInt(port)); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, client); + sharedState.connectedToMongo(connectionDetails, ryaClient); + + } catch (final IOException e) { + throw new RuntimeException("Could not connection to MongoDB. Reason: " + e.getMessage(), e); + } + + return "Connected. You must select a Rya instance to interact with next."; + } + @CliCommand(value = CONNECT_INSTANCE_CMD, help = "Connect to a specific Rya instance") public void connectToInstance( @CliOption(key = {"instance"}, mandatory = true, help = "The name of the Rya instance the shell will interact with.") diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java index ed5f261f9..a7fd802d8 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java @@ -20,14 +20,13 @@ import static java.util.Objects.requireNonNull; +import org.apache.rya.shell.SharedShellState.ShellState; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.shell.plugin.support.DefaultPromptProvider; import org.springframework.stereotype.Component; -import org.apache.rya.shell.SharedShellState.ShellState; - /** * Customizes the Rya Shell's prompt. */ @@ -50,10 +49,10 @@ public String getPrompt() { case DISCONNECTED: return "rya> "; case CONNECTED_TO_STORAGE: - return String.format("rya/%s> ", state.getConnectionDetails().get().getInstanceName()); + return String.format("rya/%s> ", state.getAccumuloDetails().get().getInstanceName()); case CONNECTED_TO_INSTANCE: return String.format("rya/%s:%s> ", - state.getConnectionDetails().get().getInstanceName(), + state.getAccumuloDetails().get().getInstanceName(), state.getRyaInstanceName().get()); default: return "rya> "; diff --git a/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java b/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java index 526b03149..6c8a57c8b 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java @@ -23,17 +23,18 @@ import java.util.Objects; import java.util.concurrent.locks.ReentrantLock; -import edu.umd.cs.findbugs.annotations.Nullable; +import org.apache.rya.api.client.RyaClient; +import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; +import org.apache.rya.api.client.mongo.MongoConnectionDetails; + +import com.google.common.base.Optional; + import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import net.jcip.annotations.Immutable; import net.jcip.annotations.ThreadSafe; -import com.google.common.base.Optional; - -import org.apache.rya.api.client.RyaClient; -import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; - /** * Holds values that are shared between the various Rya command classes. */ @@ -86,7 +87,7 @@ public void connectedToAccumulo( // Store the connection details. shellState = ShellState.builder() .setConnectionState( ConnectionState.CONNECTED_TO_STORAGE ) - .setAccumuloConnectionDetails( connectionDetails ) + .setAccumuloDetails( connectionDetails ) .setConnectedCommands( connectedCommands ) .build(); } finally { @@ -94,6 +95,40 @@ public void connectedToAccumulo( } } + /** + * This method indicates a shift into the {@link ConnectionState#CONNECTED_TO_STORAGE} state. + *

+ * Store the values used by a Mongo Rya Storage connection. This may only be called when + * the shell is disconnected. + * + * @param connectionDetails - Metadata about the Mongo connection. (not null) + * @param connectedCommands - Rya Commands that will execute against the Mongo database. (not null) + * @throws IllegalStateException Thrown if the shell is already connected to a Rya storage. + */ + public void connectedToMongo( + final MongoConnectionDetails connectionDetails, + final RyaClient connectedCommands) throws IllegalStateException { + requireNonNull(connectionDetails); + requireNonNull(connectedCommands); + + lock.lock(); + try { + // Ensure the Rya Shell is disconnected. + if(shellState.getConnectionState() != ConnectionState.DISCONNECTED) { + throw new IllegalStateException("You must clear the old connection state before you may set a new connection state."); + } + + // Store the connection details. + shellState = ShellState.builder() + .setConnectionState( ConnectionState.CONNECTED_TO_STORAGE ) + .setMongoDetails( connectionDetails ) + .setConnectedCommands( connectedCommands ) + .build(); + } finally { + lock.unlock(); + } + } + /** * This method indicates a shift into the {@link ConnectionState#CONNECTED_TO_INSTANCE} state. *

@@ -159,6 +194,21 @@ public static enum ConnectionState { CONNECTED_TO_INSTANCE; } + /** + * Enumerates the various types of Rya Storages the shell may connect to. + */ + public static enum StorageType { + /** + * The Rya instances are stored in Accumulo. + */ + ACCUMULO, + + /** + * The Rya instances are stored in MongoDB. + */ + MONGO; + } + /** * Values that define the state of a Rya Shell. */ @@ -169,7 +219,9 @@ public static final class ShellState { private final ConnectionState connectionState; // Connection specific values. - private final Optional connectionDetails; + private final Optional storageType; + private final Optional accumuloDetails; + private final Optional mongoDetails; private final Optional connectedCommands; // Instance specific values. @@ -177,11 +229,15 @@ public static final class ShellState { private ShellState( final ConnectionState connectionState, - final Optional connectionDetails, + final Optional storageType, + final Optional accumuloDetails, + final Optional mongoDetails, final Optional connectedCommands, final Optional instanceName) { this.connectionState = requireNonNull(connectionState); - this.connectionDetails = requireNonNull(connectionDetails); + this.storageType = requireNonNull(storageType); + this.accumuloDetails = requireNonNull(accumuloDetails); + this.mongoDetails = requireNonNull(mongoDetails); this.connectedCommands = requireNonNull(connectedCommands); this.instanceName = requireNonNull(instanceName); } @@ -193,12 +249,29 @@ public ConnectionState getConnectionState() { return connectionState; } + /** + * @return The type of storage the shell is connected to if it is connected to a storage. + */ + public Optional getStorageType() { + return storageType; + } + /** * @return Metadata about the Accumulo connection. The value will not be present - * if the Rya Shell is not connected to a storage. + * if the Rya Shell is not connected to a storage or is connected to another type + * of storage. */ - public Optional getConnectionDetails() { - return connectionDetails; + public Optional getAccumuloDetails() { + return accumuloDetails; + } + + /** + * @return Metadata about the Mongo connection. The value will not be present + * if the Rya Shell is not connected to a storage or is connected to another type + * of storage. + */ + public Optional getMongoDetails() { + return mongoDetails; } /** @@ -220,7 +293,7 @@ public Optional getRyaInstanceName() { @Override public int hashCode() { - return Objects.hash(connectionState, connectionDetails, connectedCommands, instanceName); + return Objects.hash(connectionState, accumuloDetails, connectedCommands, instanceName); } @Override @@ -231,7 +304,7 @@ public boolean equals(final Object obj) { if(obj instanceof ShellState) { final ShellState state = (ShellState)obj; return Objects.equals(connectionState, state.connectionState) && - Objects.equals(connectionDetails, state.connectionDetails) && + Objects.equals(accumuloDetails, state.accumuloDetails) && Objects.equals(connectedCommands, state.connectedCommands) && Objects.equals(instanceName, state.instanceName); } @@ -264,7 +337,9 @@ public static class Builder { private ConnectionState connectionState; // Connection specific values. - private AccumuloConnectionDetails connectionDetails; + private StorageType storageType; + private AccumuloConnectionDetails accumuloDetails; + private MongoConnectionDetails mongoDetails; private RyaClient connectedCommands; // Instance specific values. @@ -283,7 +358,7 @@ public Builder() { } */ public Builder(final ShellState shellState) { this.connectionState = shellState.getConnectionState(); - this.connectionDetails = shellState.getConnectionDetails().orNull(); + this.accumuloDetails = shellState.getAccumuloDetails().orNull(); this.connectedCommands = shellState.getConnectedCommands().orNull(); this.instanceName = shellState.getRyaInstanceName().orNull(); } @@ -298,11 +373,30 @@ public Builder setConnectionState(@Nullable final ConnectionState connectionStat } /** - * @param connectionDetails - Metadata about the Accumulo connection. + * @param accumuloDetails - Metadata about the Accumulo connection. + * @return This {@link Builder} so that method invocations may be chained. + */ + public Builder setAccumuloDetails(@Nullable final AccumuloConnectionDetails accumuloDetails) { + // If we are setting the details, clear any old ones. + if(accumuloDetails != null) { + this.storageType = StorageType.ACCUMULO; + this.mongoDetails = null; + } + this.accumuloDetails = accumuloDetails; + return this; + } + + /** + * @param mongoDetails - Metadata about the Mongo connection. * @return This {@link Builder} so that method invocations may be chained. */ - public Builder setAccumuloConnectionDetails(@Nullable final AccumuloConnectionDetails connectionDetails) { - this.connectionDetails = connectionDetails; + public Builder setMongoDetails(@Nullable final MongoConnectionDetails mongoDetails) { + // If we are setting the details, clear any old ones. + if(mongoDetails != null) { + this.storageType = StorageType.MONGO; + this.accumuloDetails = null; + } + this.mongoDetails = mongoDetails; return this; } @@ -330,7 +424,9 @@ public Builder setRyaInstanceName(@Nullable final String instanceName) { public ShellState build() { return new ShellState( connectionState, - Optional.fromNullable(connectionDetails), + Optional.fromNullable(storageType), + Optional.fromNullable(accumuloDetails), + Optional.fromNullable(mongoDetails), Optional.fromNullable(connectedCommands), Optional.fromNullable(instanceName)); } diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java index 6e21f8dbd..d8d5b8f15 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaAdminCommandsTest.java @@ -48,6 +48,7 @@ import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.client.Uninstall; import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; +import org.apache.rya.api.client.mongo.MongoConnectionDetails; import org.apache.rya.api.instance.RyaDetails; import org.apache.rya.api.instance.RyaDetails.EntityCentricIndexDetails; import org.apache.rya.api.instance.RyaDetails.FreeTextIndexDetails; @@ -354,7 +355,7 @@ public void install() throws DuplicateInstanceNameException, RyaClientException, } @Test - public void installWithParameters() throws DuplicateInstanceNameException, RyaClientException, IOException { + public void installWithAccumuloParameters() throws DuplicateInstanceNameException, RyaClientException, IOException { // Mock the object that performs the install operation. final Install mockInstall = mock(Install.class); @@ -368,7 +369,6 @@ public void installWithParameters() throws DuplicateInstanceNameException, RyaCl final boolean enableTableHashPrefix = false; final boolean enableEntityCentricIndex = true; final boolean enableFreeTextIndex = false; - final boolean enableGeospatialIndex = true; final boolean enableTemporalIndex = false; final boolean enablePcjIndex = true; final String fluoPcjAppName = instanceName + "pcj_updater"; @@ -378,7 +378,6 @@ public void installWithParameters() throws DuplicateInstanceNameException, RyaCl .setEnableTableHashPrefix(enableTableHashPrefix) .setEnableEntityCentricIndex(enableEntityCentricIndex) .setEnableFreeTextIndex(enableFreeTextIndex) - .setEnableGeoIndex(enableGeospatialIndex) .setEnableTemporalIndex(enableTemporalIndex) .setEnablePcjIndex(enablePcjIndex) .setFluoPcjAppName(fluoPcjAppName) @@ -390,7 +389,7 @@ public void installWithParameters() throws DuplicateInstanceNameException, RyaCl when(mockInstallPrompt.promptVerified(eq(instanceName), eq(installConfig))).thenReturn(true); final RyaAdminCommands commands = new RyaAdminCommands(state, mockInstallPrompt, mock(SparqlPrompt.class), mock(UninstallPrompt.class)); - final String message = commands.installWithParameters(instanceName, enableTableHashPrefix, enableEntityCentricIndex, enableFreeTextIndex, enableGeospatialIndex, enableTemporalIndex, enablePcjIndex, fluoPcjAppName); + final String message = commands.installWithAccumuloParameters(instanceName, enableTableHashPrefix, enableEntityCentricIndex, enableFreeTextIndex, enableTemporalIndex, enablePcjIndex, fluoPcjAppName); // Verify the values that were provided to the command were passed through to the Install. verify(mockInstall).install(eq(instanceName), eq(installConfig)); @@ -401,7 +400,7 @@ public void installWithParameters() throws DuplicateInstanceNameException, RyaCl } @Test - public void installWithParameters_userAbort() throws DuplicateInstanceNameException, RyaClientException, IOException { + public void installWithAccumuloParameters_userAbort() throws DuplicateInstanceNameException, RyaClientException, IOException { // Mock the object that performs the install operation. final Install mockInstall = mock(Install.class); @@ -415,7 +414,6 @@ public void installWithParameters_userAbort() throws DuplicateInstanceNameExcept final boolean enableTableHashPrefix = false; final boolean enableEntityCentricIndex = true; final boolean enableFreeTextIndex = false; - final boolean enableGeospatialIndex = true; final boolean enableTemporalIndex = false; final boolean enablePcjIndex = true; final String fluoPcjAppName = instanceName + "pcj_updater"; @@ -425,7 +423,6 @@ public void installWithParameters_userAbort() throws DuplicateInstanceNameExcept .setEnableTableHashPrefix(enableTableHashPrefix) .setEnableEntityCentricIndex(enableEntityCentricIndex) .setEnableFreeTextIndex(enableFreeTextIndex) - .setEnableGeoIndex(enableGeospatialIndex) .setEnableTemporalIndex(enableTemporalIndex) .setEnablePcjIndex(enablePcjIndex) .setFluoPcjAppName(fluoPcjAppName) @@ -437,13 +434,50 @@ public void installWithParameters_userAbort() throws DuplicateInstanceNameExcept when(mockInstallPrompt.promptVerified(eq(instanceName), eq(installConfig))).thenReturn(false); final RyaAdminCommands commands = new RyaAdminCommands(state, mockInstallPrompt, mock(SparqlPrompt.class), mock(UninstallPrompt.class)); - final String message = commands.installWithParameters(instanceName, enableTableHashPrefix, enableEntityCentricIndex, enableFreeTextIndex, enableGeospatialIndex, enableTemporalIndex, enablePcjIndex, fluoPcjAppName); + final String message = commands.installWithAccumuloParameters(instanceName, enableTableHashPrefix, enableEntityCentricIndex, enableFreeTextIndex, enableTemporalIndex, enablePcjIndex, fluoPcjAppName); // Verify a message is returned that indicates the success of the operation. final String expected = "Skipping Installation."; assertEquals(expected, message); } + @Test + public void installWithMongoParameters() throws DuplicateInstanceNameException, RyaClientException, IOException { + // Mock the object that performs the install operation. + final Install mockInstall = mock(Install.class); + + final RyaClient mockCommands = mock(RyaClient.class); + when(mockCommands.getInstall()).thenReturn( mockInstall ); + + final SharedShellState state = new SharedShellState(); + state.connectedToMongo(mock(MongoConnectionDetails.class), mockCommands); + + final String instanceName = "unitTests"; + final boolean enableFreeTextIndex = false; + final boolean enableTemporalIndex = false; + + // Execute the command. + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableFreeTextIndex(enableFreeTextIndex) + .setEnableTemporalIndex(enableTemporalIndex) + .build(); + + final InstallPrompt mockInstallPrompt = mock(InstallPrompt.class); + when(mockInstallPrompt.promptInstanceName()).thenReturn( instanceName ); + when(mockInstallPrompt.promptInstallConfiguration(instanceName)).thenReturn( installConfig ); + when(mockInstallPrompt.promptVerified(eq(instanceName), eq(installConfig))).thenReturn(true); + + final RyaAdminCommands commands = new RyaAdminCommands(state, mockInstallPrompt, mock(SparqlPrompt.class), mock(UninstallPrompt.class)); + final String message = commands.installWithMongoParameters(instanceName, enableFreeTextIndex, enableTemporalIndex); + + // Verify the values that were provided to the command were passed through to the Install. + verify(mockInstall).install(eq(instanceName), eq(installConfig)); + + // Verify a message is returned that indicates the success of the operation. + final String expected = "The Rya instance named 'unitTests' has been installed."; + assertEquals(expected, message); + } + @Test public void listInstances() throws RyaClientException, IOException { // Mock the object that performs the list operation. diff --git a/extras/shell/src/test/java/org/apache/rya/shell/SharedShellStateTest.java b/extras/shell/src/test/java/org/apache/rya/shell/SharedShellStateTest.java index e79d1863a..b5f136c35 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/SharedShellStateTest.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/SharedShellStateTest.java @@ -21,12 +21,11 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; -import org.junit.Test; - import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; import org.apache.rya.shell.SharedShellState.ConnectionState; import org.apache.rya.shell.SharedShellState.ShellState; +import org.junit.Test; /** * Tests the methods of {@link SharedShellState}. @@ -57,7 +56,7 @@ public void disconnectedToConnectedToStorage() { // Verify the state. final ShellState expected = ShellState.builder() .setConnectionState(ConnectionState.CONNECTED_TO_STORAGE) - .setAccumuloConnectionDetails(connectionDetails) + .setAccumuloDetails(connectionDetails) .setConnectedCommands(connectedCommands) .build(); @@ -92,7 +91,7 @@ public void connectedToInstance() { // Verify the state. final ShellState expected = ShellState.builder() .setConnectionState(ConnectionState.CONNECTED_TO_INSTANCE) - .setAccumuloConnectionDetails(connectionDetails) + .setAccumuloDetails(connectionDetails) .setConnectedCommands(connectedCommands) .setRyaInstanceName("instance") .build(); @@ -118,7 +117,7 @@ public void ConnectedToInstanceAgain() { // Verify the state. final ShellState expected = ShellState.builder() .setConnectionState(ConnectionState.CONNECTED_TO_INSTANCE) - .setAccumuloConnectionDetails(connectionDetails) + .setAccumuloDetails(connectionDetails) .setConnectedCommands(connectedCommands) .setRyaInstanceName("secondInstance") .build(); From 5bc98c4445953124dd78c5cb5c8579d133c8b39c Mon Sep 17 00:00:00 2001 From: "David W. Lotts" Date: Tue, 19 Dec 2017 15:39:31 -0500 Subject: [PATCH 06/13] RYA-104 --- .../MongoRyaInstanceDetailsRepository.java | 1 + .../client/mongo/MongoConnectionDetails.java | 49 +++---- .../client/mongo/MongoExecuteSparqlQuery.java | 122 +++++++++++++++++- .../rya/api/client/mongo/MongoInstall.java | 6 + .../api/client/mongo/MongoLoadStatements.java | 55 +++++++- .../client/mongo/MongoLoadStatementsFile.java | 6 +- .../mongo/MongoExecuteSparqlQueryIT.java | 41 ++++-- .../mongo/MongoGetInstanceDetailsIT.java | 9 +- .../rya/api/client/mongo/MongoInstallIT.java | 32 +++-- .../client/mongo/MongoInstanceExistsIT.java | 10 +- .../client/mongo/MongoListInstancesIT.java | 10 +- .../mongo/MongoLoadStatementsFileIT.java | 60 +++------ .../client/mongo/MongoLoadStatementsIT.java | 98 +++++++++++++- .../api/client/mongo/MongoUninstallIT.java | 10 +- .../rya/shell/RyaConnectionCommands.java | 4 +- .../apache/rya/shell/RyaPromptProvider.java | 19 ++- .../apache/rya/shell/SharedShellState.java | 2 + .../rya/shell/RyaConnectionCommandsIT.java | 47 +++++++ .../rya/shell/RyaPromptProviderTest.java | 22 +++- 19 files changed, 475 insertions(+), 128 deletions(-) diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java index 9bb558e86..1eadea894 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/instance/MongoRyaInstanceDetailsRepository.java @@ -56,6 +56,7 @@ public class MongoRyaInstanceDetailsRepository implements RyaDetailsRepository { public MongoRyaInstanceDetailsRepository(final MongoClient client, final String instanceName) { checkNotNull(client); this.instanceName = requireNonNull( instanceName ); + // the rya instance is the Mongo db name. This ignores any collection-prefix. db = client.getDB(this.instanceName); } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index d7922892a..b348d344a 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -30,7 +30,7 @@ import edu.umd.cs.findbugs.annotations.NonNull; /** - * The information the shell used to connect to Mongo DB. + * The information the shell used to connect to Mongo server, not the DB or collections. */ @DefaultAnnotation(NonNull.class) public class MongoConnectionDetails { @@ -48,30 +48,30 @@ public class MongoConnectionDetails { * @param hostname - The hostname of the Mongo DB that was connected to. (not null) * @param port - The port of the Mongo DB that was connected to. */ - public MongoConnectionDetails( - final String username, - final char[] userPass, - final String hostname, - final int port) { + public MongoConnectionDetails( // + final String username, // + final char[] userPass, // + final String hostname, // + final int port) { this.username = requireNonNull(username); this.userPass = requireNonNull(userPass); this.hostname = requireNonNull(hostname); this.port = port; } - /** - * @return The username that was used to establish the connection. - */ - public String getUsername() { - return this.username; - } - - /** - * @return The password that was used to establish the connection. - */ - public char[] getPassword() { - return this.userPass; - } + // /** + // * @return The username that was used to establish the connection. + // */ + // public String getUsername() { + // return this.username; + // } + // + // /** + // * @return The password that was used to establish the connection. + // */ + // public char[] getPassword() { + // return this.userPass; + // } /** * @return The hostname of the Mongo DB that was connected to. @@ -90,7 +90,8 @@ public int getPort() { /** * Create a {@link MongoDBRdfConfiguration} that is using this object's values. * - * @param ryaInstanceName - The Rya instance to connect to. + * @param ryaInstanceName + * - The Rya instance to connect to. * @return Constructs a new {@link MongoDBRdfConfiguration} object with values from this object. */ public MongoDBRdfConfiguration build(final String ryaInstanceName) { @@ -105,13 +106,13 @@ public MongoDBRdfConfiguration build(final String ryaInstanceName, final MongoCl conf.setBoolean(ConfigUtils.USE_MONGO, true); conf.setMongoInstance(hostname); conf.setMongoPort("" + port); - conf.setMongoUser(username); - conf.setMongoPassword(new String(userPass)); + // conf.setMongoUser(username); + // conf.setMongoPassword(new String(userPass)); conf.setMongoDBName(ryaInstanceName); // Both of these are ways to configure the collection prefixes. - conf.setCollectionName(ryaInstanceName); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, ryaInstanceName); + conf.setCollectionName("rya"); + conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya"); if (mongoClient != null) { conf.setMongoClient(mongoClient); } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java index f51d42dbc..681e34418 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java @@ -20,9 +20,34 @@ import static java.util.Objects.requireNonNull; +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.text.DecimalFormat; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.log4j.Logger; import org.apache.rya.api.client.ExecuteSparqlQuery; import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.persist.RyaDAOException; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import org.apache.rya.rdftriplestore.inference.InferenceEngineException; +import org.apache.rya.sail.config.RyaSailFactory; +import org.openrdf.query.BindingSet; +import org.openrdf.query.MalformedQueryException; +import org.openrdf.query.QueryEvaluationException; +import org.openrdf.query.QueryLanguage; +import org.openrdf.query.TupleQuery; +import org.openrdf.query.TupleQueryResultHandlerException; +import org.openrdf.query.resultio.text.csv.SPARQLResultsCSVWriter; +import org.openrdf.repository.RepositoryException; +import org.openrdf.repository.sail.SailRepository; +import org.openrdf.repository.sail.SailRepositoryConnection; +import org.openrdf.sail.Sail; +import org.openrdf.sail.SailException; import com.mongodb.MongoClient; @@ -30,27 +55,114 @@ import edu.umd.cs.findbugs.annotations.NonNull; /** - * TODO impl, test, doc + * Execute a sparql query on mongo Rya. */ @DefaultAnnotation(NonNull.class) public class MongoExecuteSparqlQuery extends MongoCommand implements ExecuteSparqlQuery { + private static final Logger log = Logger.getLogger(MongoExecuteSparqlQuery.class); + private final InstanceExists instanceExists; + /** - * Constructs an instance of {@link }. + * Constructs an instance. * * @param connectionDetails - Details about the values that were used to create the client. (not null) * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ public MongoExecuteSparqlQuery(final MongoConnectionDetails connectionDetails, final MongoClient client) { super(connectionDetails, client); + instanceExists = new MongoInstanceExists(connectionDetails, client); } - @Override public String executeSparqlQuery(final String ryaInstanceName, final String sparqlQuery) throws InstanceDoesNotExistException, RyaClientException { requireNonNull(ryaInstanceName); requireNonNull(sparqlQuery); + requireNonNull(ryaInstanceName); + requireNonNull(sparqlQuery); + + // Ensure the Rya Instance exists. + if (!instanceExists.exists(ryaInstanceName)) { + throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", + ryaInstanceName)); + } + Sail sail = null; + SailRepository sailRepo = null; + SailRepositoryConnection sailRepoConn = null; + // Get a Sail object that is connected to the Rya instance. + final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName, getClient()); + // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. + try { + sail = RyaSailFactory.getInstance(ryaConf); + } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException + | AccumuloSecurityException e) { + throw new RyaClientException("While getting a sail instance.", e); + } + + // Load the file. + sailRepo = new SailRepository(sail); + try { + sailRepoConn = sailRepo.getConnection(); + + // Execute the query. + final long start = System.currentTimeMillis(); + final TupleQuery tupleQuery = sailRepoConn.prepareTupleQuery(QueryLanguage.SPARQL, sparqlQuery); + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + final CountingSPARQLResultsCSVWriter handler = new CountingSPARQLResultsCSVWriter(baos); + tupleQuery.evaluate(handler); + final StringBuilder sb = new StringBuilder(); + + final String newline = "\n"; + sb.append("Query Result:").append(newline); + sb.append(new String(baos.toByteArray(), StandardCharsets.UTF_8)); + + final String seconds = new DecimalFormat("0.0##").format((System.currentTimeMillis() - start) / 1000.0); + sb.append("Retrieved ").append(handler.getCount()).append(" results in ").append(seconds).append( + " seconds."); + + return sb.toString(); + + // } catch (final SailException | AccumuloException | AccumuloSecurityException | RyaDAOException + // | InferenceEngineException e) { + // throw new RyaClientException("A problem connecting to the Rya instance named '" + ryaInstanceName + // + "' has caused the query to fail.", e); + } catch (final MalformedQueryException e) { + throw new RyaClientException("There was a problem parsing the supplied query.", e); + } catch (final QueryEvaluationException | TupleQueryResultHandlerException e) { + throw new RyaClientException("There was a problem evaluating the supplied query.", e); + } catch (final RepositoryException e) { + throw new RyaClientException("There was a problem executing the query against the Rya instance named " + + ryaInstanceName + ".", e); + } finally { + // close anything? + } + } + + /** + * Subclasses {@link SPARQLResultsCSVWriter} to keep track of the total count of handled {@link BindingSet} objects. + */ + private static class CountingSPARQLResultsCSVWriter extends SPARQLResultsCSVWriter { + + private int count = 0; + + /** + * @param out - The OutputStream for results to be written to. + */ + public CountingSPARQLResultsCSVWriter(final OutputStream out) { + super(out); + } + + @Override + public void handleSolution(final BindingSet bindingSet) throws TupleQueryResultHandlerException { + super.handleSolution(bindingSet); + count++; + } - // TODO Auto-generated method stub - return null; + /** + * + * @return The number of BindingSets that were handled by {@link #handleSolution(BindingSet)}. + */ + public int getCount() { + return count; + } } } \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java index bcd79f6b7..9718cf1b2 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java @@ -52,6 +52,12 @@ /** * An Mongo implementation of the {@link Install} command. + * Note about the scheme: + * the Rya instance is used as the mongoDBName. + * the Rya triples, instance details, and Rya indexes each get their own collection. + * the triples collection name is always constant: "rya_triples" (or? ryaInstance+"_triples") + * This means that each Mongo DB can have only one Rya instance. + * A Collection corresponds to an Accumulo table. */ @DefaultAnnotation(NonNull.class) public class MongoInstall extends MongoCommand implements Install { diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java index 846adfe5c..9a323559b 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java @@ -18,10 +18,25 @@ */ package org.apache.rya.api.client.mongo; +import static java.util.Objects.requireNonNull; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.log4j.Logger; import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.LoadStatements; import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.api.persist.RyaDAOException; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import org.apache.rya.rdftriplestore.inference.InferenceEngineException; +import org.apache.rya.sail.config.RyaSailFactory; import org.openrdf.model.Statement; +import org.openrdf.repository.RepositoryException; +import org.openrdf.repository.sail.SailRepository; +import org.openrdf.repository.sail.SailRepositoryConnection; +import org.openrdf.sail.Sail; +import org.openrdf.sail.SailException; import com.mongodb.MongoClient; @@ -33,15 +48,53 @@ */ @DefaultAnnotation(NonNull.class) public class MongoLoadStatements extends MongoCommand implements LoadStatements { + private static final Logger log = Logger.getLogger(MongoLoadStatements.class); + private final InstanceExists instanceExists; + /** + * Constructs an instance. + * + * @param connectionDetails - Details to connect to the server. (not null) + * @param client - Provides programmatic access to the instance of Mongo + * that hosts Rya instance. (not null) + */ public MongoLoadStatements(MongoConnectionDetails connectionDetails, MongoClient client) { super(connectionDetails, client); + instanceExists = new MongoInstanceExists(connectionDetails, client); } @Override public void loadStatements(String ryaInstanceName, Iterable statements) throws InstanceDoesNotExistException, RyaClientException { - // TODO Auto-generated method stub + requireNonNull(ryaInstanceName); + requireNonNull(statements); + + // Ensure the Rya Instance exists. + if (!instanceExists.exists(ryaInstanceName)) { + throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", + ryaInstanceName)); + } + + Sail sail = null; + SailRepository sailRepo = null; + SailRepositoryConnection sailRepoConn = null; + // Get a Sail object that is connected to the Rya instance. + final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName, getClient()); + // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. + try { + sail = RyaSailFactory.getInstance(ryaConf); + } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException + | AccumuloSecurityException e) { + throw new RyaClientException("While getting a sail instance.", e); + } + // Load the file. + sailRepo = new SailRepository(sail); + try { + sailRepoConn = sailRepo.getConnection(); + sailRepoConn.add(statements); + } catch (RepositoryException e) { + throw new RyaClientException("While getting a connection and adding statements.", e); + } } } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java index 9f689444c..e57b6cb87 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java @@ -59,10 +59,8 @@ public class MongoLoadStatementsFile extends MongoCommand implements LoadStateme /** * Constructs an instance of {@link MongoListInstances}. * - * @param connectionDetails - * - Details to connect to the server. (not null) - * @param client - * - Provides programmatic access to the instance of Mongo + * @param connectionDetails - Details to connect to the server. (not null) + * @param client - Provides programmatic access to the instance of Mongo * that hosts Rya instance. (not null) */ public MongoLoadStatementsFile(MongoConnectionDetails connectionDetails, MongoClient client) { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java index 153992c4c..ce866dd8d 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java @@ -18,12 +18,17 @@ */ package org.apache.rya.api.client.mongo; -import org.apache.rya.api.client.Install; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.rya.api.client.ExecuteSparqlQuery; import org.apache.rya.api.client.Install.DuplicateInstanceNameException; -import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.mongodb.MongoTestBase; import org.junit.Test; +import org.openrdf.model.Statement; import com.mongodb.MongoException; @@ -33,18 +38,38 @@ public class MongoExecuteSparqlQueryIT extends MongoTestBase { @Test public void ExecuteSparqlQuery_exec() throws MongoException, DuplicateInstanceNameException, RyaClientException { + // Install an instance of Rya. MongoConnectionDetails connectionDetails = getConnectionDetails(); - // Install a few instances of Rya using the install command. - final Install install = new MongoInstall(connectionDetails, getMongoClient()); - install.install("instanceExec", InstallConfiguration.builder().build()); - MongoExecuteSparqlQuery executeSparql = new MongoExecuteSparqlQuery(connectionDetails, getMongoClient()); - // TODO executeSparql. + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + // install rya and load some data + final List loadMe = MongoLoadStatementsIT.installAndLoad(); + // Here comes the method to test + ExecuteSparqlQuery executeSparql = ryaClient.getExecuteSparqlQuery(); + final String sparql = "SELECT * where { ?a ?b ?c }"; + String results = executeSparql.executeSparqlQuery(conf.getMongoDBName(), sparql); + System.out.println(results); + assertTrue("result has header.", results.startsWith("Query Result:")); + assertTrue("result has column headings.", results.contains("a,b,c")); + assertTrue("result has footer.", results.contains("Retrieved 3 results in")); + for (Statement expect : loadMe) { + assertTrue("All results should contain expected subjects:", + results.contains(expect.getSubject().stringValue())); + assertTrue("All results should contain expected predicates:", + results.contains(expect.getPredicate().stringValue())); + assertTrue("All results should contain expected objects:", + results.contains(expect.getObject().stringValue())); + } } /** * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - return new MongoConnectionDetails(conf.getMongoUser(), conf.getMongoPassword().toCharArray(), conf.getMongoInstance(), Integer.parseInt(conf.getMongoPort())); + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt(conf.getMongoPort())); + } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java index 4c70edaba..9c8e6988d 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java @@ -114,10 +114,9 @@ public void getDetails_instanceDoesNotHaveDetails() throws MongoException, Table * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt( conf.getMongoPort() )); + return new MongoConnectionDetails(conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java index 1ec7e9e7b..08c472d9e 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java @@ -18,6 +18,7 @@ */ package org.apache.rya.api.client.mongo; +import static org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -29,6 +30,7 @@ import org.apache.rya.api.client.Install.DuplicateInstanceNameException; import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.mongodb.MongoTestBase; import org.junit.Test; @@ -41,14 +43,10 @@ public class MongoInstallIT extends MongoTestBase { @Test public void install() throws DuplicateInstanceNameException, RyaClientException { // Install an instance of Rya. - final String ryaInstance = conf.getCollectionName(); + final String ryaInstance = conf.getMongoDBName(); // Setup the connection details that were used for the embedded Mongo DB instance we are testing with. - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt( conf.getMongoPort() )); + final MongoConnectionDetails connectionDetails = getConnectionDetails(); // Check that the instance does not exist. final InstanceExists instanceExists = new MongoInstanceExists(connectionDetails, this.getMongoClient()); @@ -61,17 +59,19 @@ public void install() throws DuplicateInstanceNameException, RyaClientException .setEnableTemporalIndex(true) .build(); - final Install install = new MongoInstall(connectionDetails, this.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final Install install = ryaClient.getInstall(); + assertTrue("ryaClient should give mongoInstall", install instanceof MongoInstall); install.install(ryaInstance, installConfig); // Check that the instance exists. assertTrue(instanceExists.exists(ryaInstance)); // Show that the expected collections were created within the database. - final List expected = Arrays.asList("instance_details", ryaInstance + "_triples"); + final List expected = Arrays.asList(INSTANCE_DETAILS_COLLECTION_NAME, "rya_triples"); int count = 0; final List found = new ArrayList<>(); - for (final String collection : this.getMongoClient().getDatabase(ryaInstance).listCollectionNames()) { + for (final String collection : this.getMongoClient().getDatabase(conf.getMongoDBName()).listCollectionNames()) { count += expected.contains(collection) ? 1 : 0; found.add( collection ); } @@ -85,11 +85,7 @@ public void install_alreadyExists() throws DuplicateInstanceNameException, RyaCl final String instanceName = conf.getCollectionName(); final InstallConfiguration installConfig = InstallConfiguration.builder().build(); - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt( conf.getMongoPort() )); + final MongoConnectionDetails connectionDetails = getConnectionDetails(); final Install install = new MongoInstall(connectionDetails, this.getMongoClient()); install.install(instanceName, installConfig); @@ -97,4 +93,12 @@ public void install_alreadyExists() throws DuplicateInstanceNameException, RyaCl // Install it again throws expected error. install.install(instanceName, installConfig); } + + private MongoConnectionDetails getConnectionDetails() { + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt(conf.getMongoPort())); + } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java index 6611f7f55..1b2f4991f 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java @@ -70,11 +70,11 @@ public void doesNotExist() throws MongoException { /** * @return copy from conf to MongoConnectionDetails */ - private MongoConnectionDetails getConnectionDetails() { + private MongoConnectionDetails getConnectionDetails() {// return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt( conf.getMongoPort() )); + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java index 561434d1e..3307fa7d2 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java @@ -59,11 +59,11 @@ public void listInstances_hasRyaDetailsTable() throws MongoException, DuplicateI /** * @return copy from conf to MongoConnectionDetails */ - private MongoConnectionDetails getConnectionDetails() { + private MongoConnectionDetails getConnectionDetails() {// return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt( conf.getMongoPort() )); + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index c3b7cd0a1..c144aba38 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -44,7 +44,7 @@ public class MongoLoadStatementsFileIT extends MongoTestBase { @Test(expected = InstanceDoesNotExistException.class) public void instanceDoesNotExist() throws Exception { - + org.apache.log4j.BasicConfigurator.configure(); final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getHostname(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); } @@ -52,26 +52,23 @@ public void instanceDoesNotExist() throws Exception { @Test public void loadTurtleFile() throws Exception { // Install an instance of Rya. - final InstallConfiguration installConfig = InstallConfiguration.builder()// - .setEnableTableHashPrefix(false)// - .setEnableEntityCentricIndex(false)// - .setEnableFreeTextIndex(false)// - .setEnableTemporalIndex(false)// - .setEnablePcjIndex(false)// - .setEnableGeoIndex(false)// + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(false) + .setEnableEntityCentricIndex(false) + .setEnableFreeTextIndex(false) + .setEnableTemporalIndex(false) + .setEnablePcjIndex(false) + .setEnableGeoIndex(false) .build(); MongoConnectionDetails connectionDetails = getConnectionDetails(); final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); final Install install = ryaClient.getInstall(); - install.install(conf.getCollectionName(), installConfig); + install.install(conf.getMongoDBName(), installConfig); // Load the test statement file. - ryaClient.getLoadStatementsFile() - // LoadStatementsFile loadStatementsFile = new MongoLoadStatementsFile(connectionDetails, getMongoClient()); - // loadStatementsFile - .loadStatements( // - conf.getCollectionName(), // - Paths.get("src/test/resources/example.ttl"), // + ryaClient.getLoadStatementsFile().loadStatements( + conf.getMongoDBName(), + Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); // Verify that the statements were loaded. @@ -87,31 +84,10 @@ public void loadTurtleFile() throws Exception { System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); while (x.hasNext()) { Document y = x.next(); - System.out.println("getRyaCollection()=" + y); + statements.add(vf.createStatement(vf.createURI(y.getString("subject")), vf.createURI(y.getString("predicate")), vf.createURI(y.getString("object")))); } assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); - // final WholeRowTripleResolver tripleResolver = new WholeRowTripleResolver(); - // final Scanner scanner = getConnector().createScanner(getRyaInstanceName() + "spo", new Authorizations()); - // final Iterator> it = scanner.iterator(); - // while(it.hasNext()) { - // final Entry next = it.next(); - // - // final Key key = next.getKey(); - // final byte[] row = key.getRow().getBytes(); - // final byte[] columnFamily = key.getColumnFamily().getBytes(); - // final byte[] columnQualifier = key.getColumnQualifier().getBytes(); - // final TripleRow tripleRow = new TripleRow(row, columnFamily, columnQualifier); - // - // final RyaStatement ryaStatement = tripleResolver.deserialize(TABLE_LAYOUT.SPO, tripleRow); - // final Statement statement = RyaToRdfConversions.convertStatement(ryaStatement); - // - // // Filter out the rya version statement if it is present. - // if(!isRyaMetadataStatement(vf, statement)) { - // statements.add( statement ); - // } - // } - // - // assertEquals(expected, statements); + assertEquals("All rows in DB should match expected rows:", expected, statements); } private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { @@ -122,10 +98,10 @@ private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement st * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - return new MongoConnectionDetails(// - conf.getMongoUser(), // - conf.getMongoPassword().toCharArray(), // - conf.getMongoInstance(), // + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), Integer.parseInt(conf.getMongoPort())); } } diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java index fb616bccb..43d9017e0 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java @@ -18,9 +18,105 @@ */ package org.apache.rya.api.client.mongo; +import static org.junit.Assert.assertEquals; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.rya.api.client.Install; +import org.apache.rya.api.client.Install.DuplicateInstanceNameException; +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClient; +import org.apache.rya.api.client.RyaClientException; +import org.apache.rya.mongodb.MongoTestBase; +import org.bson.Document; +import org.junit.Test; +import org.openrdf.model.Statement; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.ValueFactoryImpl; + +import com.mongodb.client.MongoCursor; + /** * Integration tests the methods of {@link MongoLoadStatements}. */ -public class MongoLoadStatementsIT { +public class MongoLoadStatementsIT extends MongoTestBase { + @Test(expected = InstanceDoesNotExistException.class) + public void instanceDoesNotExist() throws Exception { + org.apache.log4j.BasicConfigurator.configure(); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); + // Skip the install step to create error causing situation. + ryaClient.getLoadStatements().loadStatements(getConnectionDetails().getHostname(), makeTestStatements()); + } + + /** + * Pass a list of statements to our loadStatement class. + * + * @throws Exception + */ + @Test + public void loadTurtleFile() throws Exception { + // Install an instance of Rya. + final List loadMe = installAndLoad(); + final List stmtResults = new ArrayList<>(); + final MongoCursor triplesIterator = getRyaCollection().find().iterator(); + final ValueFactory vf = new ValueFactoryImpl(); + while (triplesIterator.hasNext()) { + Document triple = triplesIterator.next(); + stmtResults.add(vf.createStatement(vf.createURI(triple.getString("subject")), vf.createURI(triple.getString( + "predicate")), vf.createURI(triple.getString("object")))); + } + stmtResults.sort(((stmt1, stmt2) -> stmt1.getSubject().toString().compareTo(stmt2.getSubject().toString()))); + assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); + assertEquals("All rows in DB should match expected rows:", loadMe, stmtResults); + } + + /** + * @return some data to load + */ + private static List makeTestStatements() { + final List loadMe = new ArrayList<>(); + final ValueFactory vf = new ValueFactoryImpl(); + + loadMe.add(vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf + .createURI("http://example#bob"))); + loadMe.add(vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf + .createURI("http://example#charlie"))); + loadMe.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf + .createURI("http://example#icecream"))); + return loadMe; + } + public static List installAndLoad() throws DuplicateInstanceNameException, RyaClientException { + // first install rya + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(false) + .setEnableEntityCentricIndex(false) + .setEnableFreeTextIndex(false) + .setEnableTemporalIndex(false) + .setEnablePcjIndex(false) + .setEnableGeoIndex(false) + .build(); + MongoConnectionDetails connectionDetails = getConnectionDetails(); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final Install install = ryaClient.getInstall(); + install.install(conf.getMongoDBName(), installConfig); + // next, load data + final List loadMe = makeTestStatements(); + ryaClient.getLoadStatements().loadStatements( + conf.getMongoDBName(), + loadMe); + return loadMe; + } + /** + * @return copy from conf to MongoConnectionDetails + */ + private static MongoConnectionDetails getConnectionDetails() { + return new MongoConnectionDetails( + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt(conf.getMongoPort())); + } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java index dffb0f2e5..9b623a54c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java @@ -71,11 +71,11 @@ public void uninstall_instanceDoesNotExists() throws MongoException, RyaClientEx /** * @return copy from conf to MongoConnectionDetails */ - private MongoConnectionDetails getConnectionDetails() { + private MongoConnectionDetails getConnectionDetails() {// return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt( conf.getMongoPort() )); + conf.getMongoUser(), + conf.getMongoPassword().toCharArray(), + conf.getMongoInstance(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java index e51896859..7e5e6df81 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java @@ -124,8 +124,8 @@ public String printConnectionDetails() { final MongoConnectionDetails mongoDetails = sharedState.getShellState().getMongoDetails().get(); return "The shell is connected to an instance of MongoDB using the following parameters:\n" + " Hostname: " + mongoDetails.getHostname() + "\n" + - " Port: " + mongoDetails.getPort() + "\n" + - " Username:" + mongoDetails.getUsername(); + " Port: " + mongoDetails.getPort() + "\n"; + //+" Username:" + mongoDetails.getUsername(); default: throw new RuntimeException("Unrecognized StorageType: " + storageType.get()); diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java index a7fd802d8..096d5cc50 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaPromptProvider.java @@ -21,6 +21,7 @@ import static java.util.Objects.requireNonNull; import org.apache.rya.shell.SharedShellState.ShellState; +import org.apache.rya.shell.SharedShellState.StorageType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; @@ -44,16 +45,26 @@ public RyaPromptProvider(final SharedShellState sharedState) { @Override public String getPrompt() { final ShellState state = sharedState.getShellState(); - + // figure out the storage name: disconnected, mongo host, or Accumulo instance. + String storageName = "unknown"; + if (state.getStorageType().isPresent()) { + if (state.getStorageType().get() == StorageType.ACCUMULO) { + storageName = state.getAccumuloDetails().get().getInstanceName(); + } else if (state.getStorageType().get() == StorageType.MONGO) { + storageName = state.getMongoDetails().get().getHostname(); + } else { + throw new java.lang.IllegalStateException("Missing or unknown storage type."); + } + } switch(state.getConnectionState()) { case DISCONNECTED: return "rya> "; case CONNECTED_TO_STORAGE: - return String.format("rya/%s> ", state.getAccumuloDetails().get().getInstanceName()); + return String.format("rya/%s> ", storageName); case CONNECTED_TO_INSTANCE: return String.format("rya/%s:%s> ", - state.getAccumuloDetails().get().getInstanceName(), - state.getRyaInstanceName().get()); + storageName, + state.getRyaInstanceName().or("unknown")); default: return "rya> "; } diff --git a/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java b/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java index 6c8a57c8b..89cbd5d95 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java @@ -358,7 +358,9 @@ public Builder() { } */ public Builder(final ShellState shellState) { this.connectionState = shellState.getConnectionState(); + this.storageType = shellState.getStorageType().orNull(); this.accumuloDetails = shellState.getAccumuloDetails().orNull(); + this.mongoDetails = shellState.getMongoDetails().orNull(); this.connectedCommands = shellState.getConnectedCommands().orNull(); this.instanceName = shellState.getRyaInstanceName().orNull(); } diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaConnectionCommandsIT.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaConnectionCommandsIT.java index f424c49ae..45bfced3e 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaConnectionCommandsIT.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaConnectionCommandsIT.java @@ -66,6 +66,30 @@ public void connectAccumulo() throws IOException { assertTrue( connectResult.isSuccess() ); } + @Test + public void connectMongo() throws IOException { + // FIXME create mongo client/server here + final Bootstrap bootstrap = getTestBootstrap(); + final JLineShellComponent shell = getTestShell(); + + // Mock the user entering the correct password. + final ApplicationContext context = bootstrap.getApplicationContext(); + final PasswordPrompt mockPrompt = context.getBean(PasswordPrompt.class); + when(mockPrompt.getPassword()).thenReturn("password".toCharArray()); + + // Execute the connect command. + final String cmd = + RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--username root " + + "--hostname " + "localhost" + " " + + "--port " + "999"; + + final CommandResult connectResult = shell.executeCommand(cmd); + + // Ensure the connection was successful. + assertTrue(connectResult.isSuccess()); + } + @Test public void connectAccumulo_noAuths() throws IOException { final MiniAccumuloCluster cluster = getCluster(); @@ -247,4 +271,27 @@ public void disconnect() throws IOException { final CommandResult disconnectResult = shell.executeCommand( RyaConnectionCommands.DISCONNECT_COMMAND_NAME_CMD ); assertTrue( disconnectResult.isSuccess() ); } + + @Test + public void connectMongo_noAuths() throws IOException { + // FIXME make Mongo connection here + final Bootstrap bootstrap = getTestBootstrap(); + final JLineShellComponent shell = getTestShell(); + + // Mock the user entering the correct password. + final ApplicationContext context = bootstrap.getApplicationContext(); + final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class ); + when(mockPrompt.getPassword()).thenReturn("password".toCharArray()); + + // Execute the command + final String cmd = RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--username root " + + "--hostname " + "localhost" + " " + + "--port " + "999"; // FIXME + + final CommandResult connectResult = shell.executeCommand(cmd); + + // Ensure the connection was successful. + assertTrue( connectResult.isSuccess() ); + } } \ No newline at end of file diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java index d3eaa4e27..01502d2e8 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java @@ -21,10 +21,10 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; -import org.junit.Test; - import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; +import org.apache.rya.api.client.mongo.MongoConnectionDetails; +import org.junit.Test; /** * Tests the methods of {@link RyaPromptProvider}. @@ -46,7 +46,7 @@ public void notConnected() { } @Test - public void isConnected_noInstanceName() { + public void isConnected_noInstanceName_Accumulo() { // Create a shared state that is connected to a storage, but not a rya instance. final SharedShellState sharedState = new SharedShellState(); @@ -61,6 +61,22 @@ public void isConnected_noInstanceName() { assertEquals(expected, prompt); } + @Test + public void isConnected_noInstanceName_Mongo() { + // Create a shared state that is connected to a storage, but not a rya instance. + final SharedShellState sharedState = new SharedShellState(); + + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails("username", new char[] {}, "testMongoHost", 999); + sharedState.connectedToMongo(connectionDetails, mock(RyaClient.class)); + + // Create a prompt. + final String prompt = new RyaPromptProvider(sharedState).getPrompt(); + + // Verify the prompt is formatted correctly. + final String expected = "rya/testMongoHost> "; + assertEquals(expected, prompt); + } + @Test public void isConnected_hasInstanceName() { // Create a shared state that is connected to a specific instance. From c3c262d8e1944e8077be7ee6fcd30756a5edc7be Mon Sep 17 00:00:00 2001 From: isper3at Date: Wed, 27 Dec 2017 13:35:36 -0500 Subject: [PATCH 07/13] RYA-104 Post-mongo change rebase --- .../instance/MongoDetailsAdapterTest.java | 308 +++++++++--------- .../client/mongo/MongoConnectionDetails.java | 37 +-- .../rya/api/client/mongo/MongoInstall.java | 18 +- .../mongo/MongoExecuteSparqlQueryIT.java | 67 +++- .../mongo/MongoGetInstanceDetailsIT.java | 18 +- .../rya/api/client/mongo/MongoInstallIT.java | 16 +- .../client/mongo/MongoInstanceExistsIT.java | 8 +- .../client/mongo/MongoListInstancesIT.java | 8 +- .../mongo/MongoLoadStatementsFileIT.java | 34 +- .../client/mongo/MongoLoadStatementsIT.java | 46 +-- .../api/client/mongo/MongoUninstallIT.java | 8 +- extras/rya.pcj.fluo/pom.xml | 2 +- .../rya/shell/RyaConnectionCommands.java | 73 +++-- 13 files changed, 346 insertions(+), 297 deletions(-) diff --git a/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/instance/MongoDetailsAdapterTest.java b/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/instance/MongoDetailsAdapterTest.java index 533e2df4e..875f4e5fa 100644 --- a/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/instance/MongoDetailsAdapterTest.java +++ b/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/instance/MongoDetailsAdapterTest.java @@ -50,60 +50,60 @@ public class MongoDetailsAdapterTest { public void ryaDetailsToMongoTest() { // Convert the Details into a Mongo DB OBject. final RyaDetails details = RyaDetails.builder() - .setRyaInstanceName("test") - .setRyaVersion("1") - .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) - //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(true)) - .setPCJIndexDetails( - PCJIndexDetails.builder() - .setEnabled(true) - .setFluoDetails(new FluoDetails("fluo")) - .addPCJDetails( - PCJDetails.builder() - .setId("pcj_0") - .setUpdateStrategy(PCJUpdateStrategy.BATCH) - .setLastUpdateTime(new Date(0L))) - .addPCJDetails( - PCJDetails.builder() - .setId("pcj_1") - .setUpdateStrategy(PCJUpdateStrategy.BATCH) - .setLastUpdateTime(new Date(1L)))) - .setTemporalIndexDetails(new TemporalIndexDetails(true)) - .setFreeTextDetails(new FreeTextIndexDetails(true)) - .setProspectorDetails(new ProspectorDetails(Optional.fromNullable(new Date(0L)))) - .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.fromNullable(new Date(1L)))) - .build(); + .setRyaInstanceName("test") + .setRyaVersion("1") + .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) + //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(true)) + .setPCJIndexDetails( + PCJIndexDetails.builder() + .setEnabled(true) + .setFluoDetails(new FluoDetails("fluo")) + .addPCJDetails( + PCJDetails.builder() + .setId("pcj_0") + .setUpdateStrategy(PCJUpdateStrategy.BATCH) + .setLastUpdateTime(new Date(0L))) + .addPCJDetails( + PCJDetails.builder() + .setId("pcj_1") + .setUpdateStrategy(PCJUpdateStrategy.BATCH) + .setLastUpdateTime(new Date(1L)))) + .setTemporalIndexDetails(new TemporalIndexDetails(true)) + .setFreeTextDetails(new FreeTextIndexDetails(true)) + .setProspectorDetails(new ProspectorDetails(Optional.fromNullable(new Date(0L)))) + .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.fromNullable(new Date(1L)))) + .build(); final BasicDBObject actual = MongoDetailsAdapter.toDBObject(details); // Ensure it matches the expected object. final DBObject expected = (DBObject) JSON.parse( - "{ " - + "instanceName : \"test\"," - + "version : \"1\"," - + "entityCentricDetails : true," - //RYA-215 + "geoDetails : true," - + "pcjDetails : {" - + "enabled : true ," - + "fluoName : \"fluo\"," - + "pcjs : [ " - + "{" - + "id : \"pcj_0\"," - + "updateStrategy : \"BATCH\"," - + "lastUpdate : { $date : \"1970-01-01T00:00:00.000Z\"}" - + "}," - + "{" - + "id : \"pcj_1\"," - + "updateStrategy : \"BATCH\"," - + "lastUpdate : { $date : \"1970-01-01T00:00:00.001Z\"}" - + "}]" - + "}," - + "temporalDetails : true," - + "freeTextDetails : true," - + "prospectorDetails : { $date : \"1970-01-01T00:00:00.000Z\"}," - + "joinSelectivitiyDetails : { $date : \"1970-01-01T00:00:00.001Z\"}" - + "}" - ); + "{ " + + "instanceName : \"test\"," + + "version : \"1\"," + + "entityCentricDetails : true," + //RYA-215 + "geoDetails : true," + + "pcjDetails : {" + + "enabled : true ," + + "fluoName : \"fluo\"," + + "pcjs : [ " + + "{" + + "id : \"pcj_0\"," + + "updateStrategy : \"BATCH\"," + + "lastUpdate : { $date : \"1970-01-01T00:00:00.000Z\"}" + + "}," + + "{" + + "id : \"pcj_1\"," + + "updateStrategy : \"BATCH\"," + + "lastUpdate : { $date : \"1970-01-01T00:00:00.001Z\"}" + + "}]" + + "}," + + "temporalDetails : true," + + "freeTextDetails : true," + + "prospectorDetails : { $date : \"1970-01-01T00:00:00.000Z\"}," + + "joinSelectivitiyDetails : { $date : \"1970-01-01T00:00:00.001Z\"}" + + "}" + ); assertEquals(expected.toString(), actual.toString()); } @@ -112,60 +112,60 @@ public void ryaDetailsToMongoTest() { public void mongoToRyaDetailsTest() throws MalformedRyaDetailsException { // Convert the Mongo object into a RyaDetails. final BasicDBObject mongo = (BasicDBObject) JSON.parse( - "{ " - + "instanceName : \"test\"," - + "version : \"1\"," - + "entityCentricDetails : true," - //RYA-215 + "geoDetails : true," - + "pcjDetails : {" - + "enabled : true ," - + "fluoName : \"fluo\"," - + "pcjs : [ " - + "{" - + "id : \"pcj_0\"," - + "updateStrategy : \"BATCH\"," - + "lastUpdate : { $date : \"1970-01-01T00:00:00.000Z\"}" - + "}," - + "{" - + "id : \"pcj_1\"," - + "updateStrategy : \"BATCH\"," - + "lastUpdate : { $date : \"1970-01-01T00:00:00.001Z\"}" - + "}]" - + "}," - + "temporalDetails : true," - + "freeTextDetails : true," - + "prospectorDetails : { $date : \"1970-01-01T00:00:00.000Z\"}," - + "joinSelectivitiyDetails : { $date : \"1970-01-01T00:00:00.001Z\"}" - + "}" - ); + "{ " + + "instanceName : \"test\"," + + "version : \"1\"," + + "entityCentricDetails : true," + //RYA-215 + "geoDetails : true," + + "pcjDetails : {" + + "enabled : true ," + + "fluoName : \"fluo\"," + + "pcjs : [ " + + "{" + + "id : \"pcj_0\"," + + "updateStrategy : \"BATCH\"," + + "lastUpdate : { $date : \"1970-01-01T00:00:00.000Z\"}" + + "}," + + "{" + + "id : \"pcj_1\"," + + "updateStrategy : \"BATCH\"," + + "lastUpdate : { $date : \"1970-01-01T00:00:00.001Z\"}" + + "}]" + + "}," + + "temporalDetails : true," + + "freeTextDetails : true," + + "prospectorDetails : { $date : \"1970-01-01T00:00:00.000Z\"}," + + "joinSelectivitiyDetails : { $date : \"1970-01-01T00:00:00.001Z\"}" + + "}" + ); final RyaDetails actual = MongoDetailsAdapter.toRyaDetails(mongo); // Ensure it matches the expected object. final RyaDetails expected = RyaDetails.builder() - .setRyaInstanceName("test") - .setRyaVersion("1") - .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) - //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(true)) - .setPCJIndexDetails( - PCJIndexDetails.builder() - .setEnabled(true) - .setFluoDetails(new FluoDetails("fluo")) - .addPCJDetails( - PCJDetails.builder() - .setId("pcj_0") - .setUpdateStrategy(PCJUpdateStrategy.BATCH) - .setLastUpdateTime(new Date(0L))) - .addPCJDetails( - PCJDetails.builder() + .setRyaInstanceName("test") + .setRyaVersion("1") + .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) + //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(true)) + .setPCJIndexDetails( + PCJIndexDetails.builder() + .setEnabled(true) + .setFluoDetails(new FluoDetails("fluo")) + .addPCJDetails( + PCJDetails.builder() + .setId("pcj_0") + .setUpdateStrategy(PCJUpdateStrategy.BATCH) + .setLastUpdateTime(new Date(0L))) + .addPCJDetails( + PCJDetails.builder() .setId("pcj_1") .setUpdateStrategy(PCJUpdateStrategy.BATCH) .setLastUpdateTime(new Date(1L)))) - .setTemporalIndexDetails(new TemporalIndexDetails(true)) - .setFreeTextDetails(new FreeTextIndexDetails(true)) - .setProspectorDetails(new ProspectorDetails(Optional.fromNullable(new Date(0L)))) - .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.fromNullable(new Date(1L)))) - .build(); + .setTemporalIndexDetails(new TemporalIndexDetails(true)) + .setFreeTextDetails(new FreeTextIndexDetails(true)) + .setProspectorDetails(new ProspectorDetails(Optional.fromNullable(new Date(0L)))) + .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.fromNullable(new Date(1L)))) + .build(); assertEquals(expected, actual); } @@ -175,46 +175,40 @@ public void absentOptionalToRyaDetailsTest() throws MalformedRyaDetailsException // Convert the Mongo object into a RyaDetails. final BasicDBObject mongo = (BasicDBObject) JSON.parse( "{ " - + "instanceName : \"test\"," - + "version : \"1\"," - + "entityCentricDetails : true," - //RYA-215 + "geoDetails : false," - + "pcjDetails : {" - + "enabled : false," - + "fluoName : \"fluo\"," - + "pcjs : [ " - + "{" - + "id : \"pcj_1\"," - + "}" - + "]" - + "}," - + "temporalDetails : false," - + "freeTextDetails : true," - + "prospectorDetails : null," - + "joinSelectivitiyDetails : null" - + "}" - ); + + "instanceName : \"test\"," + + "version : \"1\"," + + "entityCentricDetails : true," + //RYA-215 + "geoDetails : false," + + "pcjDetails : {" + + "enabled : false," + + "pcjs : [ " + + "{" + + "id : \"pcj_1\"," + + "}" + + "]" + + "}," + + "temporalDetails : false," + + "freeTextDetails : true," + + "prospectorDetails : null," + + "joinSelectivitiyDetails : null" + + "}" + ); final RyaDetails actual = MongoDetailsAdapter.toRyaDetails(mongo); // Ensure it matches the expected object. final RyaDetails expected = RyaDetails.builder() - .setRyaInstanceName("test") - .setRyaVersion("1") - .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) - //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(false)) - .setPCJIndexDetails( - PCJIndexDetails.builder() - .setEnabled(false) - .setFluoDetails(new FluoDetails("fluo")) - .addPCJDetails( - PCJDetails.builder() - .setId("pcj_1") - .setLastUpdateTime(null))) - .setTemporalIndexDetails(new TemporalIndexDetails(false)) - .setFreeTextDetails(new FreeTextIndexDetails(true)) - .setProspectorDetails(new ProspectorDetails(Optional.absent())) - .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.absent())) - .build(); + .setRyaInstanceName("test") + .setRyaVersion("1") + .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) + //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(false)) + .setPCJIndexDetails( + PCJIndexDetails.builder() + .setEnabled(false)) + .setTemporalIndexDetails(new TemporalIndexDetails(false)) + .setFreeTextDetails(new FreeTextIndexDetails(true)) + .setProspectorDetails(new ProspectorDetails(Optional.absent())) + .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.absent())) + .build(); assertEquals(expected, actual); } @@ -223,38 +217,38 @@ public void absentOptionalToRyaDetailsTest() throws MalformedRyaDetailsException public void absentOptionalToMongoTest() { // Convert the Details into a Mongo DB OBject. final RyaDetails details = RyaDetails.builder() - .setRyaInstanceName("test") - .setRyaVersion("1") - .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) - //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(false)) - .setPCJIndexDetails( - PCJIndexDetails.builder() - .setEnabled(true) - .setFluoDetails(new FluoDetails("fluo"))) - .setTemporalIndexDetails(new TemporalIndexDetails(false)) - .setFreeTextDetails(new FreeTextIndexDetails(true)) - .setProspectorDetails(new ProspectorDetails(Optional.absent())) - .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.absent())) - .build(); + .setRyaInstanceName("test") + .setRyaVersion("1") + .setEntityCentricIndexDetails(new EntityCentricIndexDetails(true)) + //RYA-215 .setGeoIndexDetails(new GeoIndexDetails(false)) + .setPCJIndexDetails( + PCJIndexDetails.builder() + .setEnabled(true) + .setFluoDetails(new FluoDetails("fluo"))) + .setTemporalIndexDetails(new TemporalIndexDetails(false)) + .setFreeTextDetails(new FreeTextIndexDetails(true)) + .setProspectorDetails(new ProspectorDetails(Optional.absent())) + .setJoinSelectivityDetails(new JoinSelectivityDetails(Optional.absent())) + .build(); final DBObject actual = MongoDetailsAdapter.toDBObject(details); // Ensure it matches the expected object. final BasicDBObject expected = (BasicDBObject) JSON.parse( "{ " - + "instanceName : \"test\"," - + "version : \"1\"," - + "entityCentricDetails : true," - //RYA-215 + "geoDetails : false," - + "pcjDetails : {" - + "enabled : true," - + "fluoName : \"fluo\"," - + "pcjs : [ ]" - + "}," - + "temporalDetails : false," - + "freeTextDetails : true" - + "}" - ); + + "instanceName : \"test\"," + + "version : \"1\"," + + "entityCentricDetails : true," + //RYA-215 + "geoDetails : false," + + "pcjDetails : {" + + "enabled : true," + + "fluoName : \"fluo\"," + + "pcjs : [ ]" + + "}," + + "temporalDetails : false," + + "freeTextDetails : true" + + "}" + ); assertEquals(expected, actual); } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index b348d344a..cf8abc1b0 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -20,9 +20,11 @@ import static java.util.Objects.requireNonNull; -import org.apache.rya.api.RdfCloudTripleStoreConfiguration; +import java.util.Optional; + import org.apache.rya.indexing.accumulo.ConfigUtils; import org.apache.rya.mongodb.MongoDBRdfConfiguration; +import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration; import com.mongodb.MongoClient; @@ -35,26 +37,26 @@ @DefaultAnnotation(NonNull.class) public class MongoConnectionDetails { - private final String username; - private final char[] userPass; + private final Optional username; + private final Optional userPass; private final String hostname; private final int port; /** * Constructs an instance of {@link MongoConnectionDetails}. * - * @param username - The username that was used to establish the connection. (not null) - * @param password - The password that was used to establish the connection. (not null) + * @param username - The username that was used to establish the connection. + * @param password - The password that was used to establish the connection. * @param hostname - The hostname of the Mongo DB that was connected to. (not null) * @param port - The port of the Mongo DB that was connected to. */ public MongoConnectionDetails( // - final String username, // - final char[] userPass, // - final String hostname, // - final int port) { - this.username = requireNonNull(username); - this.userPass = requireNonNull(userPass); + final String username, // + final char[] userPass, // + final String hostname, // + final int port) { + this.username = Optional.ofNullable(username); + this.userPass = Optional.ofNullable(userPass); this.hostname = requireNonNull(hostname); this.port = port; } @@ -98,24 +100,21 @@ public MongoDBRdfConfiguration build(final String ryaInstanceName) { return build(ryaInstanceName, null); } - public MongoDBRdfConfiguration build(final String ryaInstanceName, final MongoClient mongoClient) { + public StatefulMongoDBRdfConfiguration build(final String ryaInstanceName, final MongoClient mongoClient) { // Note, we don't use the MongoDBRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the MongoRdfConfiguration object, it may do the right thing. final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); conf.setBoolean(ConfigUtils.USE_MONGO, true); - conf.setMongoInstance(hostname); + conf.setMongoHostname(hostname); conf.setMongoPort("" + port); // conf.setMongoUser(username); // conf.setMongoPassword(new String(userPass)); conf.setMongoDBName(ryaInstanceName); // Both of these are ways to configure the collection prefixes. - conf.setCollectionName("rya"); - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya"); - if (mongoClient != null) { - conf.setMongoClient(mongoClient); - } - return conf; + //TODO these should not be explicitly set + conf.setTablePrefix("rya"); + return new StatefulMongoDBRdfConfiguration(conf, mongoClient); } } \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java index 9718cf1b2..177355d43 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java @@ -38,8 +38,8 @@ import org.apache.rya.api.layout.TablePrefixLayoutStrategy; import org.apache.rya.api.persist.RyaDAOException; import org.apache.rya.indexing.accumulo.ConfigUtils; -import org.apache.rya.mongodb.MongoDBRdfConfiguration; import org.apache.rya.mongodb.MongoDBRyaDAO; +import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration; import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,16 +94,16 @@ public void install(final String instanceName, final InstallConfiguration instal } catch (final AlreadyInitializedException e) { // This can only happen if somebody else installs an instance of Rya with the name between the check and now. throw new DuplicateInstanceNameException("An instance of Rya has already been installed to this Rya storage "// - + "with the name '" + instanceName// - + "'. Try again with a different name."); + + "with the name '" + instanceName// + + "'. Try again with a different name."); } catch (final RyaDetailsRepositoryException e) { throw new RyaClientException("The RyaDetails couldn't be initialized. Details: " + e.getMessage(), e); } // Initialize the rest of the tables used by the Rya instance. - final MongoDBRdfConfiguration ryaConfig = makeRyaConfig(getMongoConnectionDetails(), details); + final StatefulMongoDBRdfConfiguration ryaConfig = makeRyaConfig(getMongoConnectionDetails(), details, getClient()); try { - final MongoDBRyaDAO ryaDao = new MongoDBRyaDAO(ryaConfig, getClient()); + final MongoDBRyaDAO ryaDao = new MongoDBRyaDAO(); ryaDao.setConf(ryaConfig); final TablePrefixLayoutStrategy tls = new TablePrefixLayoutStrategy(); @@ -113,7 +113,7 @@ public void install(final String instanceName, final InstallConfiguration instal ryaDao.init(); } catch (final RyaDAOException e) { throw new RyaClientException("Could not initialize all of the tables for the new Rya instance. " // - + "This instance may be left in a bad state.", e); + + "This instance may be left in a bad state.", e); } } @@ -174,11 +174,13 @@ private RyaDetails initializeRyaDetails( * * @param connectionDetails - Indicates how to connect to Mongo. (not null) * @param ryaDetails - Indicates what needs to be installed. (not null) + * @param mongoClient * @return A Rya Configuration object that can be used to perform the install. */ - private static MongoDBRdfConfiguration makeRyaConfig(final MongoConnectionDetails connectionDetails, final RyaDetails ryaDetails) { + private static StatefulMongoDBRdfConfiguration makeRyaConfig(final MongoConnectionDetails connectionDetails, final RyaDetails ryaDetails, final MongoClient mongoClient) { // Start with a configuration that is built using the connection details. - final MongoDBRdfConfiguration conf = connectionDetails.build(ryaDetails.getRyaInstanceName()); + + final StatefulMongoDBRdfConfiguration conf = connectionDetails.build(ryaDetails.getRyaInstanceName(), mongoClient); // The Mongo implementation of Rya does not currently support PCJs. if(ryaDetails.getPCJIndexDetails().isEnabled()) { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java index ce866dd8d..4e28e213c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java @@ -20,15 +20,20 @@ import static org.junit.Assert.assertTrue; +import java.util.ArrayList; import java.util.List; import org.apache.rya.api.client.ExecuteSparqlQuery; +import org.apache.rya.api.client.Install; import org.apache.rya.api.client.Install.DuplicateInstanceNameException; +import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.mongodb.MongoTestBase; import org.junit.Test; import org.openrdf.model.Statement; +import org.openrdf.model.ValueFactory; +import org.openrdf.model.impl.ValueFactoryImpl; import com.mongodb.MongoException; @@ -39,37 +44,75 @@ public class MongoExecuteSparqlQueryIT extends MongoTestBase { @Test public void ExecuteSparqlQuery_exec() throws MongoException, DuplicateInstanceNameException, RyaClientException { // Install an instance of Rya. - MongoConnectionDetails connectionDetails = getConnectionDetails(); + final MongoConnectionDetails connectionDetails = getConnectionDetails(); final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); // install rya and load some data - final List loadMe = MongoLoadStatementsIT.installAndLoad(); + final List loadMe = installAndLoad(); // Here comes the method to test - ExecuteSparqlQuery executeSparql = ryaClient.getExecuteSparqlQuery(); + final ExecuteSparqlQuery executeSparql = ryaClient.getExecuteSparqlQuery(); final String sparql = "SELECT * where { ?a ?b ?c }"; - String results = executeSparql.executeSparqlQuery(conf.getMongoDBName(), sparql); + final String results = executeSparql.executeSparqlQuery(conf.getMongoDBName(), sparql); System.out.println(results); assertTrue("result has header.", results.startsWith("Query Result:")); assertTrue("result has column headings.", results.contains("a,b,c")); assertTrue("result has footer.", results.contains("Retrieved 3 results in")); - for (Statement expect : loadMe) { + for (final Statement expect : loadMe) { assertTrue("All results should contain expected subjects:", - results.contains(expect.getSubject().stringValue())); + results.contains(expect.getSubject().stringValue())); assertTrue("All results should contain expected predicates:", - results.contains(expect.getPredicate().stringValue())); + results.contains(expect.getPredicate().stringValue())); assertTrue("All results should contain expected objects:", - results.contains(expect.getObject().stringValue())); + results.contains(expect.getObject().stringValue())); } } + private List installAndLoad() throws DuplicateInstanceNameException, RyaClientException { + // first install rya + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(false) + .setEnableEntityCentricIndex(false) + .setEnableFreeTextIndex(false) + .setEnableTemporalIndex(false) + .setEnablePcjIndex(false) + .setEnableGeoIndex(false) + .build(); + final MongoConnectionDetails connectionDetails = getConnectionDetails(); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final Install install = ryaClient.getInstall(); + install.install(conf.getMongoDBName(), installConfig); + // next, load data + final List loadMe = makeTestStatements(); + ryaClient.getLoadStatements().loadStatements( + conf.getMongoDBName(), + loadMe); + return loadMe; + } + + /** + * @return some data to load + */ + private List makeTestStatements() { + final List loadMe = new ArrayList<>(); + final ValueFactory vf = new ValueFactoryImpl(); + + loadMe.add(vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf + .createURI("http://example#bob"))); + loadMe.add(vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf + .createURI("http://example#charlie"))); + loadMe.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf + .createURI("http://example#icecream"))); + return loadMe; + } + /** * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java index 9c8e6988d..af09b490c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java @@ -59,11 +59,11 @@ public void getDetails() throws MongoException, RyaClientException { .setEnablePcjIndex(true) .build(); - final Install install = new MongoInstall(getConnectionDetails(), this.getMongoClient()); + final Install install = new MongoInstall(getConnectionDetails(), getMongoClient()); install.install(instanceName, installConfig); // Verify the correct details were persisted. - final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), this.getMongoClient()); + final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), getMongoClient()); final Optional details = getInstanceDetails.getDetails(instanceName); final RyaDetails expectedDetails = RyaDetails.builder() @@ -82,7 +82,7 @@ public void getDetails() throws MongoException, RyaClientException { // PCJJ Index is not supported, so it flips to false. .setPCJIndexDetails( PCJIndexDetails.builder() - .setEnabled(false)) + .setEnabled(false)) .setProspectorDetails( new ProspectorDetails(Optional.absent()) ) .setJoinSelectivityDetails( new JoinSelectivityDetails(Optional.absent()) ) @@ -102,10 +102,10 @@ public void getDetails_instanceDoesNotHaveDetails() throws MongoException, Table // Mimic a pre-details rya install. final String instanceName = "instance_name"; - this.getMongoClient().getDatabase(instanceName).createCollection("rya_triples"); + getMongoClient().getDatabase(instanceName).createCollection("rya_triples"); // Verify that the operation returns empty. - final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), this.getMongoClient()); + final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), getMongoClient()); final Optional details = getInstanceDetails.getDetails(instanceName); assertFalse( details.isPresent() ); } @@ -114,9 +114,9 @@ public void getDetails_instanceDoesNotHaveDetails() throws MongoException, Table * @return copy from conf to MongoConnectionDetails */ private MongoConnectionDetails getConnectionDetails() { - return new MongoConnectionDetails(conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + return new MongoConnectionDetails(conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java index 08c472d9e..2725b995f 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java @@ -49,7 +49,7 @@ public void install() throws DuplicateInstanceNameException, RyaClientException final MongoConnectionDetails connectionDetails = getConnectionDetails(); // Check that the instance does not exist. - final InstanceExists instanceExists = new MongoInstanceExists(connectionDetails, this.getMongoClient()); + final InstanceExists instanceExists = new MongoInstanceExists(connectionDetails, getMongoClient()); assertFalse(instanceExists.exists(ryaInstance)); // Install an instance of Rya with all the valid options turned on. @@ -71,7 +71,7 @@ public void install() throws DuplicateInstanceNameException, RyaClientException final List expected = Arrays.asList(INSTANCE_DETAILS_COLLECTION_NAME, "rya_triples"); int count = 0; final List found = new ArrayList<>(); - for (final String collection : this.getMongoClient().getDatabase(conf.getMongoDBName()).listCollectionNames()) { + for (final String collection : getMongoClient().getDatabase(conf.getMongoDBName()).listCollectionNames()) { count += expected.contains(collection) ? 1 : 0; found.add( collection ); } @@ -82,12 +82,12 @@ public void install() throws DuplicateInstanceNameException, RyaClientException @Test(expected = DuplicateInstanceNameException.class) public void install_alreadyExists() throws DuplicateInstanceNameException, RyaClientException { // Install an instance of Rya. - final String instanceName = conf.getCollectionName(); + final String instanceName = conf.getRyaInstanceName(); final InstallConfiguration installConfig = InstallConfiguration.builder().build(); final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final Install install = new MongoInstall(connectionDetails, this.getMongoClient()); + final Install install = new MongoInstall(connectionDetails, getMongoClient()); install.install(instanceName, installConfig); // Install it again throws expected error. @@ -96,9 +96,9 @@ public void install_alreadyExists() throws DuplicateInstanceNameException, RyaCl private MongoConnectionDetails getConnectionDetails() { return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java index 1b2f4991f..3ebfc683f 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java @@ -72,9 +72,9 @@ public void doesNotExist() throws MongoException { */ private MongoConnectionDetails getConnectionDetails() {// return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java index 3307fa7d2..062893547 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java @@ -61,9 +61,9 @@ public void listInstances_hasRyaDetailsTable() throws MongoException, DuplicateI */ private MongoConnectionDetails getConnectionDetails() {// return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index c144aba38..e3ad902a2 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -53,23 +53,23 @@ public void instanceDoesNotExist() throws Exception { public void loadTurtleFile() throws Exception { // Install an instance of Rya. final InstallConfiguration installConfig = InstallConfiguration.builder() - .setEnableTableHashPrefix(false) - .setEnableEntityCentricIndex(false) - .setEnableFreeTextIndex(false) - .setEnableTemporalIndex(false) - .setEnablePcjIndex(false) - .setEnableGeoIndex(false) - .build(); - MongoConnectionDetails connectionDetails = getConnectionDetails(); + .setEnableTableHashPrefix(false) + .setEnableEntityCentricIndex(false) + .setEnableFreeTextIndex(false) + .setEnableTemporalIndex(false) + .setEnablePcjIndex(false) + .setEnableGeoIndex(false) + .build(); + final MongoConnectionDetails connectionDetails = getConnectionDetails(); final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); final Install install = ryaClient.getInstall(); install.install(conf.getMongoDBName(), installConfig); // Load the test statement file. ryaClient.getLoadStatementsFile().loadStatements( - conf.getMongoDBName(), - Paths.get("src/test/resources/example.ttl"), - RDFFormat.TURTLE); + conf.getMongoDBName(), + Paths.get("src/test/resources/example.ttl"), + RDFFormat.TURTLE); // Verify that the statements were loaded. final ValueFactory vf = new ValueFactoryImpl(); @@ -80,10 +80,10 @@ public void loadTurtleFile() throws Exception { expected.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream"))); final List statements = new ArrayList<>(); - MongoCursor x = getRyaCollection().find().iterator(); + final MongoCursor x = getRyaCollection().find().iterator(); System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); while (x.hasNext()) { - Document y = x.next(); + final Document y = x.next(); statements.add(vf.createStatement(vf.createURI(y.getString("subject")), vf.createURI(y.getString("predicate")), vf.createURI(y.getString("object")))); } assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); @@ -99,9 +99,9 @@ private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement st */ private MongoConnectionDetails getConnectionDetails() { return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java index 43d9017e0..9dfb71b33 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java @@ -52,7 +52,7 @@ public void instanceDoesNotExist() throws Exception { /** * Pass a list of statements to our loadStatement class. - * + * * @throws Exception */ @Test @@ -63,9 +63,9 @@ public void loadTurtleFile() throws Exception { final MongoCursor triplesIterator = getRyaCollection().find().iterator(); final ValueFactory vf = new ValueFactoryImpl(); while (triplesIterator.hasNext()) { - Document triple = triplesIterator.next(); + final Document triple = triplesIterator.next(); stmtResults.add(vf.createStatement(vf.createURI(triple.getString("subject")), vf.createURI(triple.getString( - "predicate")), vf.createURI(triple.getString("object")))); + "predicate")), vf.createURI(triple.getString("object")))); } stmtResults.sort(((stmt1, stmt2) -> stmt1.getSubject().toString().compareTo(stmt2.getSubject().toString()))); assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); @@ -75,48 +75,48 @@ public void loadTurtleFile() throws Exception { /** * @return some data to load */ - private static List makeTestStatements() { + private List makeTestStatements() { final List loadMe = new ArrayList<>(); final ValueFactory vf = new ValueFactoryImpl(); loadMe.add(vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf - .createURI("http://example#bob"))); + .createURI("http://example#bob"))); loadMe.add(vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf - .createURI("http://example#charlie"))); + .createURI("http://example#charlie"))); loadMe.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf - .createURI("http://example#icecream"))); + .createURI("http://example#icecream"))); return loadMe; } - public static List installAndLoad() throws DuplicateInstanceNameException, RyaClientException { + private List installAndLoad() throws DuplicateInstanceNameException, RyaClientException { // first install rya final InstallConfiguration installConfig = InstallConfiguration.builder() - .setEnableTableHashPrefix(false) - .setEnableEntityCentricIndex(false) - .setEnableFreeTextIndex(false) - .setEnableTemporalIndex(false) - .setEnablePcjIndex(false) - .setEnableGeoIndex(false) - .build(); - MongoConnectionDetails connectionDetails = getConnectionDetails(); + .setEnableTableHashPrefix(false) + .setEnableEntityCentricIndex(false) + .setEnableFreeTextIndex(false) + .setEnableTemporalIndex(false) + .setEnablePcjIndex(false) + .setEnableGeoIndex(false) + .build(); + final MongoConnectionDetails connectionDetails = getConnectionDetails(); final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); final Install install = ryaClient.getInstall(); install.install(conf.getMongoDBName(), installConfig); // next, load data final List loadMe = makeTestStatements(); ryaClient.getLoadStatements().loadStatements( - conf.getMongoDBName(), - loadMe); + conf.getMongoDBName(), + loadMe); return loadMe; } /** * @return copy from conf to MongoConnectionDetails */ - private static MongoConnectionDetails getConnectionDetails() { + private MongoConnectionDetails getConnectionDetails() { return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java index 9b623a54c..69e3eecd3 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java @@ -73,9 +73,9 @@ public void uninstall_instanceDoesNotExists() throws MongoException, RyaClientEx */ private MongoConnectionDetails getConnectionDetails() {// return new MongoConnectionDetails( - conf.getMongoUser(), - conf.getMongoPassword().toCharArray(), - conf.getMongoInstance(), - Integer.parseInt(conf.getMongoPort())); + conf.getMongoUser(), + null,//conf.getMongoPassword().toCharArray(), + conf.getMongoHostname(), + Integer.parseInt(conf.getMongoPort())); } } \ No newline at end of file diff --git a/extras/rya.pcj.fluo/pom.xml b/extras/rya.pcj.fluo/pom.xml index b6f54033a..3cae897b2 100644 --- a/extras/rya.pcj.fluo/pom.xml +++ b/extras/rya.pcj.fluo/pom.xml @@ -37,7 +37,7 @@ pcj.fluo.api pcj.fluo.app pcj.fluo.client - pcj.fluo.integration + pcj.fluo.test.base pcj.fluo.demo diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java index 7e5e6df81..9a55460f1 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java @@ -33,7 +33,6 @@ import org.apache.rya.api.client.accumulo.AccumuloRyaClientFactory; import org.apache.rya.api.client.mongo.MongoConnectionDetails; import org.apache.rya.api.client.mongo.MongoRyaClientFactory; -import org.apache.rya.mongodb.MongoConnectorFactory; import org.apache.rya.mongodb.MongoDBRdfConfiguration; import org.apache.rya.shell.SharedShellState.ConnectionState; import org.apache.rya.shell.SharedShellState.StorageType; @@ -47,7 +46,10 @@ import org.springframework.stereotype.Component; import com.google.common.base.Optional; +import com.google.common.collect.Lists; import com.mongodb.MongoClient; +import com.mongodb.MongoCredential; +import com.mongodb.ServerAddress; /** * Spring Shell commands that manage the connection that is used by the shell. @@ -73,7 +75,7 @@ public class RyaConnectionCommands implements CommandMarker { */ @Autowired public RyaConnectionCommands(final SharedShellState state, final PasswordPrompt passwordPrompt) { - this.sharedState = requireNonNull( state ); + sharedState = requireNonNull( state ); this.passwordPrompt = requireNonNull(passwordPrompt); } @@ -90,11 +92,11 @@ public boolean areConnectCommandsAvailable() { @CliAvailabilityIndicator({CONNECT_INSTANCE_CMD}) public boolean isConnectToInstanceAvailable() { switch(sharedState.getShellState().getConnectionState()) { - case CONNECTED_TO_STORAGE: - case CONNECTED_TO_INSTANCE: - return true; - default: - return false; + case CONNECTED_TO_STORAGE: + case CONNECTED_TO_INSTANCE: + return true; + default: + return false; } } @@ -113,22 +115,22 @@ public String printConnectionDetails() { // Create a print out based on what it is connected to. switch(storageType.get()) { - case ACCUMULO: - final AccumuloConnectionDetails accDetails = sharedState.getShellState().getAccumuloDetails().get(); - return "The shell is connected to an instance of Accumulo using the following parameters:\n" + - " Username: " + accDetails.getUsername() + "\n" + - " Instance Name: " + accDetails.getInstanceName() + "\n" + - " Zookeepers: " + accDetails.getZookeepers(); - - case MONGO: - final MongoConnectionDetails mongoDetails = sharedState.getShellState().getMongoDetails().get(); - return "The shell is connected to an instance of MongoDB using the following parameters:\n" + - " Hostname: " + mongoDetails.getHostname() + "\n" + - " Port: " + mongoDetails.getPort() + "\n"; - //+" Username:" + mongoDetails.getUsername(); - - default: - throw new RuntimeException("Unrecognized StorageType: " + storageType.get()); + case ACCUMULO: + final AccumuloConnectionDetails accDetails = sharedState.getShellState().getAccumuloDetails().get(); + return "The shell is connected to an instance of Accumulo using the following parameters:\n" + + " Username: " + accDetails.getUsername() + "\n" + + " Instance Name: " + accDetails.getInstanceName() + "\n" + + " Zookeepers: " + accDetails.getZookeepers(); + + case MONGO: + final MongoConnectionDetails mongoDetails = sharedState.getShellState().getMongoDetails().get(); + return "The shell is connected to an instance of MongoDB using the following parameters:\n" + + " Hostname: " + mongoDetails.getHostname() + "\n" + + " Port: " + mongoDetails.getPort() + "\n"; + //+" Username:" + mongoDetails.getUsername(); + + default: + throw new RuntimeException("Unrecognized StorageType: " + storageType.get()); } } @@ -161,7 +163,7 @@ public String connectToAccumulo( @CliCommand(value = CONNECT_MONGO_CMD, help = "Connect the shell to an instance of MongoDB.") public String connectToMongo( - @CliOption(key = {"username"}, mandatory = true, help = "The username that will be used to connect to MongoDB.") + @CliOption(key = {"username"}, mandatory = false, help = "The username that will be used to connect to MongoDB.") final String username, @CliOption(key= {"hostname"}, mandatory = true, help = "The hostname of the MongoDB that will be connected to.") final String hostname, @@ -170,23 +172,32 @@ public String connectToMongo( // Prompt the user for their password. try { - final char[] password = passwordPrompt.getPassword(); - // Set up a configuration file that connects to the specified Mongo DB. final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); - conf.setMongoInstance(hostname); + final MongoClient client; + conf.setMongoHostname(hostname); conf.setMongoPort(port); - conf.setMongoUser(username); - conf.setMongoPassword(new String(password)); + + final char[] password; + if(username != null) { + password = passwordPrompt.getPassword(); + conf.setMongoUser(username); + conf.setMongoPassword(new String(password)); + final ServerAddress addr = new ServerAddress(hostname, Integer.parseInt(port)); + final MongoCredential creds = MongoCredential.createPlainCredential(username, "$external", password); + client = new MongoClient(addr, Lists.newArrayList(creds)); + } else { + password = null; + client = new MongoClient(hostname, Integer.parseInt(port)); + } // Create the singleton instance of Mongo that will be used through out the application. - final MongoClient client = MongoConnectorFactory.getMongoClient(conf); // Make sure the client is closed at shutdown. Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { - MongoConnectorFactory.closeMongoClient(); + client.close(); } }); From 6bb7146e642d7ce7e11c42ae2f9706b3ae359391 Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Wed, 27 Dec 2017 19:50:57 -0500 Subject: [PATCH 08/13] RYA-104 Removed the MongoCommand class. --- .../rya/api/client/mongo/MongoCommand.java | 62 ------------- .../client/mongo/MongoConnectionDetails.java | 72 +++++++-------- .../client/mongo/MongoExecuteSparqlQuery.java | 34 +++---- .../client/mongo/MongoGetInstanceDetails.java | 15 ++-- .../rya/api/client/mongo/MongoInstall.java | 89 ++++++++++--------- .../api/client/mongo/MongoInstanceExists.java | 13 +-- .../api/client/mongo/MongoListInstances.java | 20 +++-- .../api/client/mongo/MongoLoadStatements.java | 36 ++++---- .../client/mongo/MongoLoadStatementsFile.java | 25 +++--- .../client/mongo/MongoRyaClientFactory.java | 24 ++--- .../rya/api/client/mongo/MongoUninstall.java | 17 ++-- .../mongo/MongoExecuteSparqlQueryIT.java | 18 ++-- .../mongo/MongoGetInstanceDetailsIT.java | 26 +++--- .../rya/api/client/mongo/MongoInstallIT.java | 19 ++-- .../client/mongo/MongoInstanceExistsIT.java | 17 +--- .../client/mongo/MongoListInstancesIT.java | 22 +++-- .../mongo/MongoLoadStatementsFileIT.java | 24 +++-- .../client/mongo/MongoLoadStatementsIT.java | 21 ++--- .../api/client/mongo/MongoUninstallIT.java | 28 +++--- .../rya/shell/RyaConnectionCommands.java | 63 ++++++------- .../rya/shell/RyaPromptProviderTest.java | 5 +- 21 files changed, 300 insertions(+), 350 deletions(-) delete mode 100644 extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java deleted file mode 100644 index 443fec789..000000000 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoCommand.java +++ /dev/null @@ -1,62 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.api.client.mongo; - -import static java.util.Objects.requireNonNull; - -import com.mongodb.MongoClient; - -import edu.umd.cs.findbugs.annotations.DefaultAnnotation; -import edu.umd.cs.findbugs.annotations.NonNull; - -/** - * An abstract class that holds onto Mongo access information. Extend this - * when implementing a command that interacts with Mongo. - */ -@DefaultAnnotation(NonNull.class) -public abstract class MongoCommand { - - private final MongoConnectionDetails connectionDetails; - private final MongoClient client; - - /** - * Constructs an instance of {@link MongoCommand}. - * - * @param connectionDetails - Details about the values that were used to create the client. (not null) - * @param client - Provides programmatic access to the instance of Mongo that hosts Rya instance. (not null) - */ - public MongoCommand(final MongoConnectionDetails connectionDetails, final MongoClient client) { - this.connectionDetails = requireNonNull(connectionDetails); - this.client = requireNonNull(client); - } - - /** - * @return Details about the values that were used to create the connector to mongo. - */ - public MongoConnectionDetails getMongoConnectionDetails() { - return connectionDetails; - } - - /** - * @param client - Provides programmatic access to the instance of Mongo that hosts Rya instance. - */ - public MongoClient getClient() { - return client; - } -} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index cf8abc1b0..b48bb9f99 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -24,9 +24,6 @@ import org.apache.rya.indexing.accumulo.ConfigUtils; import org.apache.rya.mongodb.MongoDBRdfConfiguration; -import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration; - -import com.mongodb.MongoClient; import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -38,43 +35,31 @@ public class MongoConnectionDetails { private final Optional username; - private final Optional userPass; + private final Optional password; private final String hostname; private final int port; /** * Constructs an instance of {@link MongoConnectionDetails}. * - * @param username - The username that was used to establish the connection. - * @param password - The password that was used to establish the connection. * @param hostname - The hostname of the Mongo DB that was connected to. (not null) * @param port - The port of the Mongo DB that was connected to. + * @param username - The username that was used to establish the connection + * when performing administrative operations. (not null) + * @param password - The password that was used to establish the connection + * when performing administrative operations. (not null) */ - public MongoConnectionDetails( // - final String username, // - final char[] userPass, // - final String hostname, // - final int port) { - this.username = Optional.ofNullable(username); - this.userPass = Optional.ofNullable(userPass); + public MongoConnectionDetails( + final String hostname, + final int port, + final Optional username, + final Optional password) { this.hostname = requireNonNull(hostname); this.port = port; + this.username = requireNonNull(username); + this.password = requireNonNull(password); } - // /** - // * @return The username that was used to establish the connection. - // */ - // public String getUsername() { - // return this.username; - // } - // - // /** - // * @return The password that was used to establish the connection. - // */ - // public char[] getPassword() { - // return this.userPass; - // } - /** * @return The hostname of the Mongo DB that was connected to. */ @@ -89,18 +74,27 @@ public int getPort() { return port; } + /** + * @return The username that was used to establish the connection when performing administrative operations. + */ + public Optional getUsername() { + return this.username; + } + + /** + * @return The password that was used to establish the connection when performing administrative operations. + */ + public Optional getPassword() { + return password; + } + /** * Create a {@link MongoDBRdfConfiguration} that is using this object's values. * - * @param ryaInstanceName - * - The Rya instance to connect to. + * @param ryaInstanceName - The Rya instance to connect to. (not null) * @return Constructs a new {@link MongoDBRdfConfiguration} object with values from this object. */ public MongoDBRdfConfiguration build(final String ryaInstanceName) { - return build(ryaInstanceName, null); - } - - public StatefulMongoDBRdfConfiguration build(final String ryaInstanceName, final MongoClient mongoClient) { // Note, we don't use the MongoDBRdfConfigurationBuilder here because it explicitly sets // authorizations and visibilities to an empty string if they are not set on the builder. // If they are null in the MongoRdfConfiguration object, it may do the right thing. @@ -108,13 +102,19 @@ public StatefulMongoDBRdfConfiguration build(final String ryaInstanceName, final conf.setBoolean(ConfigUtils.USE_MONGO, true); conf.setMongoHostname(hostname); conf.setMongoPort("" + port); - // conf.setMongoUser(username); - // conf.setMongoPassword(new String(userPass)); conf.setMongoDBName(ryaInstanceName); + if(username.isPresent()) { + conf.setMongoUser(username.get()); + } + + if(password.isPresent()) { + conf.setMongoPassword( new String(password.get()) ); + } + // Both of these are ways to configure the collection prefixes. //TODO these should not be explicitly set conf.setTablePrefix("rya"); - return new StatefulMongoDBRdfConfiguration(conf, mongoClient); + return conf; } } \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java index 681e34418..d920b30fd 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java @@ -27,7 +27,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.log4j.Logger; import org.apache.rya.api.client.ExecuteSparqlQuery; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.InstanceExists; @@ -49,8 +48,6 @@ import org.openrdf.sail.Sail; import org.openrdf.sail.SailException; -import com.mongodb.MongoClient; - import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -58,20 +55,22 @@ * Execute a sparql query on mongo Rya. */ @DefaultAnnotation(NonNull.class) -public class MongoExecuteSparqlQuery extends MongoCommand implements ExecuteSparqlQuery { +public class MongoExecuteSparqlQuery implements ExecuteSparqlQuery { - private static final Logger log = Logger.getLogger(MongoExecuteSparqlQuery.class); + private final MongoConnectionDetails connectionDetails; private final InstanceExists instanceExists; - + /** * Constructs an instance. * - * @param connectionDetails - Details about the values that were used to create the client. (not null) - * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param connectionDetails - Details to connect to the server. (not null) + * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) */ - public MongoExecuteSparqlQuery(final MongoConnectionDetails connectionDetails, final MongoClient client) { - super(connectionDetails, client); - instanceExists = new MongoInstanceExists(connectionDetails, client); + public MongoExecuteSparqlQuery( + final MongoConnectionDetails connectionDetails, + final MongoInstanceExists instanceExists) { + this.connectionDetails = requireNonNull(connectionDetails); + this.instanceExists = requireNonNull(instanceExists); } @Override public String executeSparqlQuery(final String ryaInstanceName, final String sparqlQuery) throws InstanceDoesNotExistException, RyaClientException { @@ -89,12 +88,10 @@ public String executeSparqlQuery(final String ryaInstanceName, final String spar SailRepository sailRepo = null; SailRepositoryConnection sailRepoConn = null; // Get a Sail object that is connected to the Rya instance. - final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName, getClient()); - // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. + final MongoDBRdfConfiguration ryaConf = connectionDetails.build(ryaInstanceName); try { sail = RyaSailFactory.getInstance(ryaConf); - } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException - | AccumuloSecurityException e) { + } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException | AccumuloSecurityException e) { throw new RyaClientException("While getting a sail instance.", e); } @@ -116,15 +113,10 @@ public String executeSparqlQuery(final String ryaInstanceName, final String spar sb.append(new String(baos.toByteArray(), StandardCharsets.UTF_8)); final String seconds = new DecimalFormat("0.0##").format((System.currentTimeMillis() - start) / 1000.0); - sb.append("Retrieved ").append(handler.getCount()).append(" results in ").append(seconds).append( - " seconds."); + sb.append("Retrieved ").append(handler.getCount()).append(" results in ").append(seconds).append(" seconds."); return sb.toString(); - // } catch (final SailException | AccumuloException | AccumuloSecurityException | RyaDAOException - // | InferenceEngineException e) { - // throw new RyaClientException("A problem connecting to the Rya instance named '" + ryaInstanceName - // + "' has caused the query to fail.", e); } catch (final MalformedQueryException e) { throw new RyaClientException("There was a problem parsing the supplied query.", e); } catch (final QueryEvaluationException | TupleQueryResultHandlerException e) { diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java index b61e107c9..ade79908f 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetails.java @@ -40,19 +40,20 @@ * An Mongo implementation of the {@link GetInstanceDetails} command. */ @DefaultAnnotation(NonNull.class) -public class MongoGetInstanceDetails extends MongoCommand implements GetInstanceDetails { +public class MongoGetInstanceDetails implements GetInstanceDetails { + private final MongoClient adminClient; private final InstanceExists instanceExists; /** * Constructs an instance of {@link MongoGetInstanceDetails}. * - * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param adminClient - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) */ - public MongoGetInstanceDetails(final MongoConnectionDetails connectionDetails, final MongoClient client) { - super(connectionDetails, client); - instanceExists = new MongoInstanceExists(connectionDetails, client); + public MongoGetInstanceDetails(final MongoClient adminClient, MongoInstanceExists instanceExists) { + this.adminClient = requireNonNull(adminClient); + this.instanceExists = requireNonNull(instanceExists); } @Override @@ -65,7 +66,7 @@ public Optional getDetails(final String ryaInstanceName) throws Inst } // If the instance has details, then return them. - final RyaDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getClient(), ryaInstanceName); + final RyaDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(adminClient, ryaInstanceName); try { return Optional.of(detailsRepo.getRyaInstanceDetails()); } catch (final NotInitializedException e) { diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java index 177355d43..51bd4bffe 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstall.java @@ -22,6 +22,8 @@ import java.util.Date; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.rya.api.client.Install; import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.RyaClientException; @@ -35,12 +37,14 @@ import org.apache.rya.api.instance.RyaDetailsRepository; import org.apache.rya.api.instance.RyaDetailsRepository.AlreadyInitializedException; import org.apache.rya.api.instance.RyaDetailsRepository.RyaDetailsRepositoryException; -import org.apache.rya.api.layout.TablePrefixLayoutStrategy; import org.apache.rya.api.persist.RyaDAOException; import org.apache.rya.indexing.accumulo.ConfigUtils; -import org.apache.rya.mongodb.MongoDBRyaDAO; -import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration; +import org.apache.rya.mongodb.MongoDBRdfConfiguration; import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; +import org.apache.rya.rdftriplestore.inference.InferenceEngineException; +import org.apache.rya.sail.config.RyaSailFactory; +import org.openrdf.sail.Sail; +import org.openrdf.sail.SailException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,35 +56,43 @@ /** * An Mongo implementation of the {@link Install} command. - * Note about the scheme: - * the Rya instance is used as the mongoDBName. - * the Rya triples, instance details, and Rya indexes each get their own collection. - * the triples collection name is always constant: "rya_triples" (or? ryaInstance+"_triples") - * This means that each Mongo DB can have only one Rya instance. - * A Collection corresponds to an Accumulo table. + *

+ * Notes about the scheme: + *
    + *
  • The Rya instance name is used as the DB name in Mongo.
    • + *
  • The Rya triples, instance details, and Rya indexes each get their own collection.
    • + *
  • Each Mongo DB can have only one Rya instance.
    • + *
*/ @DefaultAnnotation(NonNull.class) -public class MongoInstall extends MongoCommand implements Install { +public class MongoInstall implements Install { private static final Logger log = LoggerFactory.getLogger(MongoInstall.class); + private final MongoConnectionDetails connectionDetails; + private final MongoClient adminClient; private final InstanceExists instanceExists; /** * Constructs an instance of {@link MongoInstall}. * - * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param connectionDetails - Details to connect to the server. (not null) + * @param adminClient - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) */ - public MongoInstall(final MongoConnectionDetails connectionDetails, final MongoClient client) { - super(connectionDetails, client); - instanceExists = new MongoInstanceExists(connectionDetails, client); + public MongoInstall( + final MongoConnectionDetails connectionDetails, + final MongoClient adminClient, + final MongoInstanceExists instanceExists) { + this.connectionDetails = requireNonNull(connectionDetails); + this.adminClient = requireNonNull(adminClient); + this.instanceExists = requireNonNull(instanceExists); } @Override public void install(final String instanceName, final InstallConfiguration installConfig) throws DuplicateInstanceNameException, RyaClientException { - requireNonNull(instanceName, "instanceName required."); - requireNonNull(installConfig, "installConfig required."); + requireNonNull(instanceName); + requireNonNull(installConfig); // Check to see if a Rya instance has already been installed with this name. if (instanceExists.exists(instanceName)) { @@ -88,32 +100,28 @@ public void install(final String instanceName, final InstallConfiguration instal } // Initialize the Rya Details table. - RyaDetails details; + final RyaDetails ryaDetails; try { - details = initializeRyaDetails(instanceName, installConfig); + ryaDetails = initializeRyaDetails(instanceName, installConfig); } catch (final AlreadyInitializedException e) { // This can only happen if somebody else installs an instance of Rya with the name between the check and now. - throw new DuplicateInstanceNameException("An instance of Rya has already been installed to this Rya storage "// - + "with the name '" + instanceName// - + "'. Try again with a different name."); + throw new DuplicateInstanceNameException( + "An instance of Rya has already been installed to this Rya storage " + + "with the name '" + instanceName + "'. Try again with a different name."); } catch (final RyaDetailsRepositoryException e) { throw new RyaClientException("The RyaDetails couldn't be initialized. Details: " + e.getMessage(), e); } - // Initialize the rest of the tables used by the Rya instance. - final StatefulMongoDBRdfConfiguration ryaConfig = makeRyaConfig(getMongoConnectionDetails(), details, getClient()); + // Initialize the rest of the collections used by the Rya instance. + final MongoDBRdfConfiguration ryaConfig = makeRyaConfig(connectionDetails, ryaDetails); try { - final MongoDBRyaDAO ryaDao = new MongoDBRyaDAO(); - ryaDao.setConf(ryaConfig); - - final TablePrefixLayoutStrategy tls = new TablePrefixLayoutStrategy(); - tls.setTablePrefix(instanceName); - ryaConfig.setTableLayoutStrategy(tls); - - ryaDao.init(); - } catch (final RyaDAOException e) { - throw new RyaClientException("Could not initialize all of the tables for the new Rya instance. " // - + "This instance may be left in a bad state.", e); + final Sail ryaSail = RyaSailFactory.getInstance(ryaConfig); + ryaSail.shutDown(); + } catch (final AccumuloException | AccumuloSecurityException | RyaDAOException | InferenceEngineException e) { + throw new RyaClientException("Could not initialize all of the tables for the new Rya instance. " + + "This instance may be left in a bad state.", e); + } catch (final SailException e) { + throw new RyaClientException("Problem shutting down the Sail object used to install Rya.", e); } } @@ -137,7 +145,7 @@ private String getVersion() { private RyaDetails initializeRyaDetails( final String instanceName, final InstallConfiguration installConfig) throws AlreadyInitializedException, RyaDetailsRepositoryException { - final RyaDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(getClient(), instanceName); + final RyaDetailsRepository detailsRepo = new MongoRyaInstanceDetailsRepository(adminClient, instanceName); // Build the PCJ Index details. [not supported in mongo] final PCJIndexDetails.Builder pcjDetailsBuilder = PCJIndexDetails.builder().setEnabled(false); @@ -152,7 +160,7 @@ private RyaDetails initializeRyaDetails( .setTemporalIndexDetails(new TemporalIndexDetails(installConfig.isTemporalIndexEnabled())) .setFreeTextDetails(new FreeTextIndexDetails(installConfig.isFreeTextIndexEnabled()))// - // Entity centric indexing is not supported in Mongo Db. + // Entity centric indexing is not supported in Mongo DB. .setEntityCentricIndexDetails(new EntityCentricIndexDetails(false)) .setPCJIndexDetails(pcjDetailsBuilder) @@ -174,13 +182,14 @@ private RyaDetails initializeRyaDetails( * * @param connectionDetails - Indicates how to connect to Mongo. (not null) * @param ryaDetails - Indicates what needs to be installed. (not null) - * @param mongoClient * @return A Rya Configuration object that can be used to perform the install. */ - private static StatefulMongoDBRdfConfiguration makeRyaConfig(final MongoConnectionDetails connectionDetails, final RyaDetails ryaDetails, final MongoClient mongoClient) { + private static MongoDBRdfConfiguration makeRyaConfig( + final MongoConnectionDetails connectionDetails, + final RyaDetails ryaDetails) { // Start with a configuration that is built using the connection details. - final StatefulMongoDBRdfConfiguration conf = connectionDetails.build(ryaDetails.getRyaInstanceName(), mongoClient); + final MongoDBRdfConfiguration conf = connectionDetails.build(ryaDetails.getRyaInstanceName()); // The Mongo implementation of Rya does not currently support PCJs. if(ryaDetails.getPCJIndexDetails().isEnabled()) { diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java index a00079a79..59613670b 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoInstanceExists.java @@ -31,22 +31,23 @@ * An Mongo implementation of the {@link InstanceExists} command. */ @DefaultAnnotation(NonNull.class) -public class MongoInstanceExists extends MongoCommand implements InstanceExists { +public class MongoInstanceExists implements InstanceExists { + + private final MongoClient adminClient; /** * Constructs an insatnce of {@link MongoInstanceExists}. * - * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param adminClient - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ - public MongoInstanceExists(final MongoConnectionDetails connectionDetails, final MongoClient client) { - super(connectionDetails, client); + public MongoInstanceExists(final MongoClient adminClient) { + this.adminClient = requireNonNull(adminClient); } @Override public boolean exists(final String instanceName) { requireNonNull( instanceName ); - for(final String dbName : getClient().listDatabaseNames()) { + for(final String dbName : adminClient.listDatabaseNames()) { if(dbName.equals(instanceName)) { return true; } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java index bf2984875..1154700be 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoListInstances.java @@ -18,6 +18,8 @@ */ package org.apache.rya.api.client.mongo; +import static java.util.Objects.requireNonNull; + import java.util.ArrayList; import java.util.List; @@ -34,25 +36,25 @@ * An Mongo implementation of the {@link ListInstances} command. */ @DefaultAnnotation(NonNull.class) -public class MongoListInstances extends MongoCommand implements ListInstances { +public class MongoListInstances implements ListInstances { + + private final MongoClient adminClient; + /** * Constructs an instance of {@link MongoListInstances}. * - * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param adminClient - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) */ - public MongoListInstances(final MongoConnectionDetails connectionDetails, final MongoClient client) { - super(connectionDetails, client); + public MongoListInstances(final MongoClient adminClient) { + this.adminClient = requireNonNull(adminClient); } @Override public List listInstances() throws RyaClientException { - final MongoClient client = super.getClient(); - // Each database that contains an instance details collection is a Rya Instance. final List ryaInstances = new ArrayList<>(); - for (final String db : getClient().listDatabaseNames()) { - for(final String collection : client.getDatabase(db).listCollectionNames()) { + for (final String db : adminClient.listDatabaseNames()) { + for(final String collection : adminClient.getDatabase(db).listCollectionNames()) { if (collection.equals(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME)) { ryaInstances.add(db); break; diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java index 9a323559b..37f1d95ee 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java @@ -22,7 +22,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.log4j.Logger; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.LoadStatements; @@ -38,8 +37,6 @@ import org.openrdf.sail.Sail; import org.openrdf.sail.SailException; -import com.mongodb.MongoClient; - import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -47,44 +44,42 @@ * An Mongo implementation of the {@link LoadStatements} command. */ @DefaultAnnotation(NonNull.class) -public class MongoLoadStatements extends MongoCommand implements LoadStatements { - private static final Logger log = Logger.getLogger(MongoLoadStatements.class); +public class MongoLoadStatements implements LoadStatements { + + private final MongoConnectionDetails connectionDetails; private final InstanceExists instanceExists; /** * Constructs an instance. * * @param connectionDetails - Details to connect to the server. (not null) - * @param client - Provides programmatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) */ - public MongoLoadStatements(MongoConnectionDetails connectionDetails, MongoClient client) { - super(connectionDetails, client); - instanceExists = new MongoInstanceExists(connectionDetails, client); + public MongoLoadStatements(final MongoConnectionDetails connectionDetails, final MongoInstanceExists instanceExists) { + this.connectionDetails = requireNonNull(connectionDetails); + this.instanceExists = requireNonNull(instanceExists); } @Override - public void loadStatements(String ryaInstanceName, Iterable statements) throws InstanceDoesNotExistException, RyaClientException { + public void loadStatements(final String ryaInstanceName, final Iterable statements) throws InstanceDoesNotExistException, RyaClientException { requireNonNull(ryaInstanceName); requireNonNull(statements); // Ensure the Rya Instance exists. if (!instanceExists.exists(ryaInstanceName)) { - throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", - ryaInstanceName)); + throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", ryaInstanceName)); } Sail sail = null; SailRepository sailRepo = null; SailRepositoryConnection sailRepoConn = null; + // Get a Sail object that is connected to the Rya instance. - final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName, getClient()); - // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. + final MongoDBRdfConfiguration ryaConf = connectionDetails.build(ryaInstanceName); try { sail = RyaSailFactory.getInstance(ryaConf); - } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException - | AccumuloSecurityException e) { - throw new RyaClientException("While getting a sail instance.", e); + } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException | AccumuloSecurityException e) { + throw new RyaClientException("Could not load statements into Rya because of a problem while creating the Sail object.", e); } // Load the file. @@ -92,9 +87,8 @@ public void loadStatements(String ryaInstanceName, Iterable try { sailRepoConn = sailRepo.getConnection(); sailRepoConn.add(statements); - } catch (RepositoryException e) { + } catch (final RepositoryException e) { throw new RyaClientException("While getting a connection and adding statements.", e); } } - -} +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java index e57b6cb87..00a7c61e0 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFile.java @@ -25,7 +25,6 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.log4j.Logger; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.LoadStatementsFile; @@ -42,8 +41,6 @@ import org.openrdf.sail.Sail; import org.openrdf.sail.SailException; -import com.mongodb.MongoClient; - import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -51,25 +48,26 @@ * An Mongo implementation of the {@link LoadStatementsFile} command. */ @DefaultAnnotation(NonNull.class) -public class MongoLoadStatementsFile extends MongoCommand implements LoadStatementsFile { - private static final Logger log = Logger.getLogger(MongoLoadStatementsFile.class); +public class MongoLoadStatementsFile implements LoadStatementsFile { + private final MongoConnectionDetails connectionDetails; private final InstanceExists instanceExists; /** * Constructs an instance of {@link MongoListInstances}. * * @param connectionDetails - Details to connect to the server. (not null) - * @param client - Provides programmatic access to the instance of Mongo - * that hosts Rya instance. (not null) + * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) */ - public MongoLoadStatementsFile(MongoConnectionDetails connectionDetails, MongoClient client) { - super(connectionDetails, client); - instanceExists = new MongoInstanceExists(connectionDetails, client); + public MongoLoadStatementsFile( + final MongoConnectionDetails connectionDetails, + final MongoInstanceExists instanceExists) { + this.connectionDetails = requireNonNull(connectionDetails); + this.instanceExists = requireNonNull(instanceExists); } @Override - public void loadStatements(String ryaInstanceName, Path statementsFile, RDFFormat format) throws InstanceDoesNotExistException, RyaClientException { + public void loadStatements(final String ryaInstanceName, final Path statementsFile, final RDFFormat format) throws InstanceDoesNotExistException, RyaClientException { requireNonNull(ryaInstanceName); requireNonNull(statementsFile); requireNonNull(format); @@ -83,8 +81,7 @@ public void loadStatements(String ryaInstanceName, Path statementsFile, RDFForma SailRepository sailRepo = null; SailRepositoryConnection sailRepoConn = null; // Get a Sail object that is connected to the Rya instance. - final MongoDBRdfConfiguration ryaConf = getMongoConnectionDetails().build(ryaInstanceName, getClient()); - // ryaConf.setFlush(false); //Accumulo version said: RYA-327 should address this hardcoded value. + final MongoDBRdfConfiguration ryaConf = connectionDetails.build(ryaInstanceName); try { sail = RyaSailFactory.getInstance(ryaConf); } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException | AccumuloSecurityException e) { @@ -100,4 +97,4 @@ public void loadStatements(String ryaInstanceName, Path statementsFile, RDFForma throw new RyaClientException("While getting a connection and adding statements from a file.", e); } } -} +} \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java index f4d68e720..55c6e8ac2 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoRyaClientFactory.java @@ -41,32 +41,34 @@ public class MongoRyaClientFactory { * Rya that is hosted by a MongoDB server. * * @param connectionDetails - Details about the values that were used to connect to Mongo DB. (not null) - * @param mongoClient - The MongoDB client the commands will use. (not null) + * @param adminClient - The MongoDB client the administrative commands will use. (not null) * @return The initialized commands. */ public static RyaClient build( final MongoConnectionDetails connectionDetails, - final MongoClient mongoClient) { + final MongoClient adminClient) { requireNonNull(connectionDetails); - requireNonNull(mongoClient); + requireNonNull(adminClient); // Build the RyaCommands option with the initialized commands. + final MongoInstanceExists instanceExists = new MongoInstanceExists(adminClient); + return new RyaClient( - new MongoInstall(connectionDetails, mongoClient), + new MongoInstall(connectionDetails, adminClient, instanceExists), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), - new MongoGetInstanceDetails(connectionDetails, mongoClient), - new MongoInstanceExists(connectionDetails, mongoClient), - new MongoListInstances(connectionDetails, mongoClient), + new MongoGetInstanceDetails(adminClient, instanceExists), + instanceExists, + new MongoListInstances(adminClient), Optional.empty(), Optional.empty(), - new MongoUninstall(connectionDetails, mongoClient), - new MongoLoadStatements(connectionDetails, mongoClient), - new MongoLoadStatementsFile(connectionDetails, mongoClient), - new MongoExecuteSparqlQuery(connectionDetails, mongoClient)); + new MongoUninstall(adminClient, instanceExists), + new MongoLoadStatements(connectionDetails, instanceExists), + new MongoLoadStatementsFile(connectionDetails, instanceExists), + new MongoExecuteSparqlQuery(connectionDetails, instanceExists)); } } \ No newline at end of file diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java index 979f93daf..a6fbb4c10 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java @@ -18,6 +18,8 @@ */ package org.apache.rya.api.client.mongo; +import static java.util.Objects.requireNonNull; + import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.RyaClientException; @@ -33,19 +35,20 @@ * An Mongo implementation of the {@link Uninstall} command. */ @DefaultAnnotation(NonNull.class) -public class MongoUninstall extends MongoCommand implements Uninstall { +public class MongoUninstall implements Uninstall { + private final MongoClient adminClient; private final InstanceExists instanceExists; /** * Constructs an instance of {@link MongoUninstall}. * - * @param connectionDetails - Details about the values that were used to create the connector to the cluster. (not null) - * @param connector - Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param adminClient- Provides programmatic access to the instance of Mongo that hosts Rya instances. (not null) + * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) */ - public MongoUninstall(final MongoConnectionDetails connectionDetails, final MongoClient client) { - super(connectionDetails, client); - instanceExists = new MongoInstanceExists(connectionDetails, client); + public MongoUninstall(final MongoClient adminClient, final MongoInstanceExists instanceExists) { + this.adminClient = requireNonNull(adminClient); + this.instanceExists = requireNonNull(instanceExists); } @Override @@ -54,7 +57,7 @@ public void uninstall(final String ryaInstanceName) throws InstanceDoesNotExistE if (!instanceExists.exists(ryaInstanceName)) { throw new InstanceDoesNotExistException("The database '" + ryaInstanceName + "' does not exist."); } - getClient().dropDatabase(ryaInstanceName); + adminClient.dropDatabase(ryaInstanceName); } catch (final MongoException e) { throw new RyaClientException("Failed to uninstall '" + ryaInstanceName + "' " + e.getLocalizedMessage(), e); } diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java index 4e28e213c..0be9e8b58 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java @@ -45,7 +45,7 @@ public class MongoExecuteSparqlQueryIT extends MongoTestBase { public void ExecuteSparqlQuery_exec() throws MongoException, DuplicateInstanceNameException, RyaClientException { // Install an instance of Rya. final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); // install rya and load some data final List loadMe = installAndLoad(); // Here comes the method to test @@ -77,7 +77,7 @@ private List installAndLoad() throws DuplicateInstanceNameException, .setEnableGeoIndex(false) .build(); final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); final Install install = ryaClient.getInstall(); install.install(conf.getMongoDBName(), installConfig); // next, load data @@ -104,15 +104,15 @@ private List makeTestStatements() { return loadMe; } - /** - * @return copy from conf to MongoConnectionDetails - */ private MongoConnectionDetails getConnectionDetails() { + final java.util.Optional password = conf.getMongoPassword() != null ? + java.util.Optional.of(conf.getMongoPassword().toCharArray()) : + java.util.Optional.empty(); + return new MongoConnectionDetails( - conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); - + Integer.parseInt(conf.getMongoPort()), + java.util.Optional.ofNullable(conf.getMongoUser()), + password); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java index af09b490c..e68ec49fd 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java @@ -28,6 +28,7 @@ import org.apache.rya.api.client.Install; import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceDoesNotExistException; +import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.instance.RyaDetails; import org.apache.rya.api.instance.RyaDetails.EntityCentricIndexDetails; @@ -59,11 +60,12 @@ public void getDetails() throws MongoException, RyaClientException { .setEnablePcjIndex(true) .build(); - final Install install = new MongoInstall(getConnectionDetails(), getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); + final Install install = ryaClient.getInstall(); install.install(instanceName, installConfig); // Verify the correct details were persisted. - final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), getMongoClient()); + final GetInstanceDetails getInstanceDetails = ryaClient.getGetInstanceDetails(); final Optional details = getInstanceDetails.getDetails(instanceName); final RyaDetails expectedDetails = RyaDetails.builder() @@ -93,7 +95,8 @@ public void getDetails() throws MongoException, RyaClientException { @Test(expected = InstanceDoesNotExistException.class) public void getDetails_instanceDoesNotExist() throws MongoException, RyaClientException { - final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); + final GetInstanceDetails getInstanceDetails = ryaClient.getGetInstanceDetails(); getInstanceDetails.getDetails("instance_name"); } @@ -105,18 +108,21 @@ public void getDetails_instanceDoesNotHaveDetails() throws MongoException, Table getMongoClient().getDatabase(instanceName).createCollection("rya_triples"); // Verify that the operation returns empty. - final GetInstanceDetails getInstanceDetails = new MongoGetInstanceDetails(getConnectionDetails(), getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); + final GetInstanceDetails getInstanceDetails = ryaClient.getGetInstanceDetails(); final Optional details = getInstanceDetails.getDetails(instanceName); assertFalse( details.isPresent() ); } - /** - * @return copy from conf to MongoConnectionDetails - */ private MongoConnectionDetails getConnectionDetails() { - return new MongoConnectionDetails(conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), + final java.util.Optional password = conf.getMongoPassword() != null ? + java.util.Optional.of(conf.getMongoPassword().toCharArray()) : + java.util.Optional.empty(); + + return new MongoConnectionDetails( conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); + Integer.parseInt(conf.getMongoPort()), + java.util.Optional.ofNullable(conf.getMongoUser()), + password); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java index 2725b995f..16991c766 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java @@ -25,6 +25,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Optional; import org.apache.rya.api.client.Install; import org.apache.rya.api.client.Install.DuplicateInstanceNameException; @@ -49,7 +50,7 @@ public void install() throws DuplicateInstanceNameException, RyaClientException final MongoConnectionDetails connectionDetails = getConnectionDetails(); // Check that the instance does not exist. - final InstanceExists instanceExists = new MongoInstanceExists(connectionDetails, getMongoClient()); + final InstanceExists instanceExists = new MongoInstanceExists(getMongoClient()); assertFalse(instanceExists.exists(ryaInstance)); // Install an instance of Rya with all the valid options turned on. @@ -59,9 +60,8 @@ public void install() throws DuplicateInstanceNameException, RyaClientException .setEnableTemporalIndex(true) .build(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); final Install install = ryaClient.getInstall(); - assertTrue("ryaClient should give mongoInstall", install instanceof MongoInstall); install.install(ryaInstance, installConfig); // Check that the instance exists. @@ -87,7 +87,8 @@ public void install_alreadyExists() throws DuplicateInstanceNameException, RyaCl final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final Install install = new MongoInstall(connectionDetails, getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); + final Install install = ryaClient.getInstall(); install.install(instanceName, installConfig); // Install it again throws expected error. @@ -95,10 +96,14 @@ public void install_alreadyExists() throws DuplicateInstanceNameException, RyaCl } private MongoConnectionDetails getConnectionDetails() { + final Optional password = conf.getMongoPassword() != null ? + Optional.of(conf.getMongoPassword().toCharArray()) : + Optional.empty(); + return new MongoConnectionDetails( - conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); + Integer.parseInt(conf.getMongoPort()), + Optional.ofNullable(conf.getMongoUser()), + password); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java index 3ebfc683f..a2e8e25ab 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java @@ -43,7 +43,7 @@ public void exists_ryaDetailsTable() throws MongoException, TableExistsException client.getDatabase(instanceName).createCollection(MongoRyaInstanceDetailsRepository.INSTANCE_DETAILS_COLLECTION_NAME); // Verify the command reports the instance exists. - final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), client); + final MongoInstanceExists instanceExists = new MongoInstanceExists(getMongoClient()); assertTrue( instanceExists.exists(instanceName) ); } @@ -56,25 +56,14 @@ public void exists_dataTables() throws MongoException, TableExistsException { client.getDatabase(instanceName).createCollection("rya_triples"); // Verify the command reports the instance exists. - final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), client); + final MongoInstanceExists instanceExists = new MongoInstanceExists(getMongoClient()); assertTrue( instanceExists.exists(instanceName) ); } @Test public void doesNotExist() throws MongoException { // Verify the command reports the instance does not exists. - final MongoInstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), getMongoClient()); + final MongoInstanceExists instanceExists = new MongoInstanceExists(getMongoClient()); assertFalse( instanceExists.exists("some_instance") ); } - - /** - * @return copy from conf to MongoConnectionDetails - */ - private MongoConnectionDetails getConnectionDetails() {// - return new MongoConnectionDetails( - conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), - conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); - } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java index 062893547..c66670e8f 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java @@ -22,11 +22,13 @@ import java.util.Collections; import java.util.List; +import java.util.Optional; import org.apache.rya.api.client.Install; import org.apache.rya.api.client.Install.DuplicateInstanceNameException; import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.ListInstances; +import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.mongodb.MongoTestBase; import org.junit.Test; @@ -42,13 +44,14 @@ public class MongoListInstancesIT extends MongoTestBase { @Test public void listInstances_hasRyaDetailsTable() throws MongoException, DuplicateInstanceNameException, RyaClientException { // Install a few instances of Rya using the install command. - final Install install = new MongoInstall(getConnectionDetails(), getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); + final Install install = ryaClient.getInstall(); install.install("instance1_", InstallConfiguration.builder().build()); install.install("instance2_", InstallConfiguration.builder().build()); install.install("instance3_", InstallConfiguration.builder().build()); // Fetch the list and verify it matches what is expected. - final ListInstances listInstances = new MongoListInstances(getConnectionDetails(), getMongoClient()); + final ListInstances listInstances = new MongoListInstances(getMongoClient()); final List instances = listInstances.listInstances(); Collections.sort(instances); @@ -56,14 +59,15 @@ public void listInstances_hasRyaDetailsTable() throws MongoException, DuplicateI assertEquals(expected, instances); } - /** - * @return copy from conf to MongoConnectionDetails - */ - private MongoConnectionDetails getConnectionDetails() {// + private MongoConnectionDetails getConnectionDetails() { + final Optional password = conf.getMongoPassword() != null ? + Optional.of(conf.getMongoPassword().toCharArray()) : + Optional.empty(); + return new MongoConnectionDetails( - conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); + Integer.parseInt(conf.getMongoPort()), + Optional.ofNullable(conf.getMongoUser()), + password); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index e3ad902a2..0f84fea1c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -45,7 +45,7 @@ public class MongoLoadStatementsFileIT extends MongoTestBase { @Test(expected = InstanceDoesNotExistException.class) public void instanceDoesNotExist() throws Exception { org.apache.log4j.BasicConfigurator.configure(); - final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); ryaClient.getLoadStatementsFile().loadStatements(getConnectionDetails().getHostname(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); } @@ -61,7 +61,7 @@ public void loadTurtleFile() throws Exception { .setEnableGeoIndex(false) .build(); final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); final Install install = ryaClient.getInstall(); install.install(conf.getMongoDBName(), installConfig); @@ -81,7 +81,6 @@ public void loadTurtleFile() throws Exception { final List statements = new ArrayList<>(); final MongoCursor x = getRyaCollection().find().iterator(); - System.out.println("getRyaCollection().count()=" + getRyaCollection().count()); while (x.hasNext()) { final Document y = x.next(); statements.add(vf.createStatement(vf.createURI(y.getString("subject")), vf.createURI(y.getString("predicate")), vf.createURI(y.getString("object")))); @@ -90,18 +89,15 @@ public void loadTurtleFile() throws Exception { assertEquals("All rows in DB should match expected rows:", expected, statements); } - private boolean isRyaMetadataStatement(final ValueFactory vf, final Statement statement) { - return statement.getPredicate().equals(vf.createURI("urn:org.apache.rya/2012/05#version")) || statement.getPredicate().equals(vf.createURI("urn:org.apache.rya/2012/05#rts")); - } - - /** - * @return copy from conf to MongoConnectionDetails - */ private MongoConnectionDetails getConnectionDetails() { + final java.util.Optional password = conf.getMongoPassword() != null ? + java.util.Optional.of(conf.getMongoPassword().toCharArray()) : + java.util.Optional.empty(); + return new MongoConnectionDetails( - conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); + Integer.parseInt(conf.getMongoPort()), + java.util.Optional.ofNullable(conf.getMongoUser()), + password); } -} +} \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java index 9dfb71b33..47b953da0 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java @@ -42,18 +42,17 @@ * Integration tests the methods of {@link MongoLoadStatements}. */ public class MongoLoadStatementsIT extends MongoTestBase { + @Test(expected = InstanceDoesNotExistException.class) public void instanceDoesNotExist() throws Exception { org.apache.log4j.BasicConfigurator.configure(); - final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); // Skip the install step to create error causing situation. ryaClient.getLoadStatements().loadStatements(getConnectionDetails().getHostname(), makeTestStatements()); } /** * Pass a list of statements to our loadStatement class. - * - * @throws Exception */ @Test public void loadTurtleFile() throws Exception { @@ -99,7 +98,7 @@ private List installAndLoad() throws DuplicateInstanceNameException, .setEnableGeoIndex(false) .build(); final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); final Install install = ryaClient.getInstall(); install.install(conf.getMongoDBName(), installConfig); // next, load data @@ -109,14 +108,16 @@ private List installAndLoad() throws DuplicateInstanceNameException, loadMe); return loadMe; } - /** - * @return copy from conf to MongoConnectionDetails - */ + private MongoConnectionDetails getConnectionDetails() { + final java.util.Optional password = conf.getMongoPassword() != null ? + java.util.Optional.of(conf.getMongoPassword().toCharArray()) : + java.util.Optional.empty(); + return new MongoConnectionDetails( - conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); + Integer.parseInt(conf.getMongoPort()), + java.util.Optional.ofNullable(conf.getMongoUser()), + password); } } \ No newline at end of file diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java index 69e3eecd3..6c9d4fd29 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java @@ -21,10 +21,13 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import java.util.Optional; + import org.apache.rya.api.client.Install; import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.InstanceExists; +import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.client.Uninstall; import org.apache.rya.mongodb.MongoTestBase; @@ -43,15 +46,16 @@ public void uninstall() throws MongoException, RyaClientException { final String instanceName = "testInstance_"; final InstallConfiguration installConfig = InstallConfiguration.builder().setEnableTableHashPrefix(true).setEnableEntityCentricIndex(true).setEnableFreeTextIndex(true).setEnableTemporalIndex(true).setEnablePcjIndex(true).setEnableGeoIndex(true).setFluoPcjAppName("fluo_app_name").build(); - final Install install = new MongoInstall(getConnectionDetails(), conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); + final Install install = ryaClient.getInstall(); install.install(instanceName, installConfig); // Show that the instance exists. - final InstanceExists instanceExists = new MongoInstanceExists(getConnectionDetails(), conf.getMongoClient()); + final InstanceExists instanceExists = ryaClient.getInstanceExists(); assertTrue( instanceExists.exists(instanceName) ); // Uninstall the instance - final Uninstall uninstall = new MongoUninstall(getConnectionDetails(), conf.getMongoClient()); + final Uninstall uninstall = ryaClient.getUninstall(); uninstall.uninstall(instanceName); // Check that the instance no longer exists. @@ -64,18 +68,20 @@ public void uninstall_instanceDoesNotExists() throws MongoException, RyaClientEx final String instanceName = "testInstance_"; // Uninstall the instance - final Uninstall uninstall = new MongoUninstall(getConnectionDetails(), conf.getMongoClient()); + final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); + final Uninstall uninstall = ryaClient.getUninstall(); uninstall.uninstall(instanceName); } - /** - * @return copy from conf to MongoConnectionDetails - */ - private MongoConnectionDetails getConnectionDetails() {// + private MongoConnectionDetails getConnectionDetails() { + final Optional password = conf.getMongoPassword() != null ? + Optional.of(conf.getMongoPassword().toCharArray()) : + Optional.empty(); + return new MongoConnectionDetails( - conf.getMongoUser(), - null,//conf.getMongoPassword().toCharArray(), conf.getMongoHostname(), - Integer.parseInt(conf.getMongoPort())); + Integer.parseInt(conf.getMongoPort()), + Optional.ofNullable(conf.getMongoUser()), + password); } } \ No newline at end of file diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java index 9a55460f1..4cb14fa28 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.CharBuffer; +import java.util.Optional; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; @@ -33,7 +34,6 @@ import org.apache.rya.api.client.accumulo.AccumuloRyaClientFactory; import org.apache.rya.api.client.mongo.MongoConnectionDetails; import org.apache.rya.api.client.mongo.MongoRyaClientFactory; -import org.apache.rya.mongodb.MongoDBRdfConfiguration; import org.apache.rya.shell.SharedShellState.ConnectionState; import org.apache.rya.shell.SharedShellState.StorageType; import org.apache.rya.shell.util.ConnectorFactory; @@ -45,11 +45,7 @@ import org.springframework.shell.core.annotation.CliOption; import org.springframework.stereotype.Component; -import com.google.common.base.Optional; -import com.google.common.collect.Lists; import com.mongodb.MongoClient; -import com.mongodb.MongoCredential; -import com.mongodb.ServerAddress; /** * Spring Shell commands that manage the connection that is used by the shell. @@ -108,7 +104,7 @@ public boolean isDisconnectAvailable() { @CliCommand(value = PRINT_CONNECTION_DETAILS_CMD, help = "Print information about the Shell's Rya storage connection.") public String printConnectionDetails() { // Check to see if the shell is connected to any storages. - final Optional storageType = sharedState.getShellState().getStorageType(); + final com.google.common.base.Optional storageType = sharedState.getShellState().getStorageType(); if(!storageType.isPresent()) { return "The shell is not connected to anything."; } @@ -124,10 +120,17 @@ public String printConnectionDetails() { case MONGO: final MongoConnectionDetails mongoDetails = sharedState.getShellState().getMongoDetails().get(); - return "The shell is connected to an instance of MongoDB using the following parameters:\n" + - " Hostname: " + mongoDetails.getHostname() + "\n" + - " Port: " + mongoDetails.getPort() + "\n"; - //+" Username:" + mongoDetails.getUsername(); + + StringBuilder message = new StringBuilder() + .append("The shell is connected to an instance of MongoDB using the following parameters:\\n") + .append(" Hostname: " + mongoDetails.getHostname() + "\n") + .append(" Port: " + mongoDetails.getPort() + "\n"); + + if(mongoDetails.getUsername().isPresent()) { + message.append(" Username:" + mongoDetails.getUsername().get()); + } + + return message.toString(); default: throw new RuntimeException("Unrecognized StorageType: " + storageType.get()); @@ -163,47 +166,41 @@ public String connectToAccumulo( @CliCommand(value = CONNECT_MONGO_CMD, help = "Connect the shell to an instance of MongoDB.") public String connectToMongo( - @CliOption(key = {"username"}, mandatory = false, help = "The username that will be used to connect to MongoDB.") + @CliOption(key = {"username"}, mandatory = false, help = + "The username that will be used to connect to MongoDB when performing administrative tasks.") final String username, @CliOption(key= {"hostname"}, mandatory = true, help = "The hostname of the MongoDB that will be connected to.") final String hostname, @CliOption(key= {"port"}, mandatory = true, help = "The port of the MongoDB that will be connected to.") final String port) { - // Prompt the user for their password. try { - // Set up a configuration file that connects to the specified Mongo DB. - final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration(); - final MongoClient client; - conf.setMongoHostname(hostname); - conf.setMongoPort(port); - - final char[] password; + // If a username was provided, then prompt for a password. + char[] password = null; if(username != null) { password = passwordPrompt.getPassword(); - conf.setMongoUser(username); - conf.setMongoPassword(new String(password)); - final ServerAddress addr = new ServerAddress(hostname, Integer.parseInt(port)); - final MongoCredential creds = MongoCredential.createPlainCredential(username, "$external", password); - client = new MongoClient(addr, Lists.newArrayList(creds)); - } else { - password = null; - client = new MongoClient(hostname, Integer.parseInt(port)); } - // Create the singleton instance of Mongo that will be used through out the application. + // Create the Mongo Connection Details that describe the Mongo DB Server we are interacting with. + final MongoConnectionDetails connectionDetails = new MongoConnectionDetails( + hostname, + Integer.parseInt(port), + Optional.ofNullable(username), + Optional.ofNullable(password)); + + // Connect to a MongoDB server. TODO Figure out how to provide auth info? + MongoClient adminClient = new MongoClient(hostname, Integer.parseInt(port)); // Make sure the client is closed at shutdown. Runtime.getRuntime().addShutdownHook(new Thread() { @Override public void run() { - client.close(); + adminClient.close(); } }); // Initialize the connected to Mongo shared state. - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails(username, password, hostname, Integer.parseInt(port)); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, client); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, adminClient); sharedState.connectedToMongo(connectionDetails, ryaClient); } catch (final IOException e) { @@ -218,6 +215,10 @@ public void connectToInstance( @CliOption(key = {"instance"}, mandatory = true, help = "The name of the Rya instance the shell will interact with.") final String instance) { try { + // TODO When you are connected to mongo db, then connecting to an instance may require + // a username/password. this is because each Database has its own credentials, so + // every rya instance likewise has their own credentials. + final InstanceExists instanceExists = sharedState.getShellState().getConnectedCommands().get().getInstanceExists(); // TODO gracefully fail if that version doen't support it. maybe the list command should go ahead diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java index 01502d2e8..559aae922 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaPromptProviderTest.java @@ -21,6 +21,8 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; +import java.util.Optional; + import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.accumulo.AccumuloConnectionDetails; import org.apache.rya.api.client.mongo.MongoConnectionDetails; @@ -66,7 +68,8 @@ public void isConnected_noInstanceName_Mongo() { // Create a shared state that is connected to a storage, but not a rya instance. final SharedShellState sharedState = new SharedShellState(); - final MongoConnectionDetails connectionDetails = new MongoConnectionDetails("username", new char[] {}, "testMongoHost", 999); + final MongoConnectionDetails connectionDetails = + new MongoConnectionDetails("testMongoHost", 999, Optional.of("username"), Optional.of(new char[] {})); sharedState.connectedToMongo(connectionDetails, mock(RyaClient.class)); // Create a prompt. From 5739f78e568cf0696d4773521b658bd196a0aabe Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Thu, 28 Dec 2017 18:21:44 -0500 Subject: [PATCH 09/13] RYA-104 Closing resources and fixing tests for some of the Mongo interactors. --- .../client/mongo/MongoExecuteSparqlQuery.java | 75 +++++++------- .../api/client/mongo/MongoLoadStatements.java | 43 +++++--- .../client/mongo/MongoLoadStatementsFile.java | 39 ++++++-- .../mongo/MongoExecuteSparqlQueryIT.java | 50 ++++------ .../mongo/MongoLoadStatementsFileIT.java | 30 +++--- .../client/mongo/MongoLoadStatementsIT.java | 99 ++++++++++--------- .../rya/shell/RyaConnectionCommands.java | 17 ++-- .../apache/rya/shell/SharedShellState.java | 24 +++++ 8 files changed, 224 insertions(+), 153 deletions(-) diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java index d920b30fd..d3b7bcb64 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQuery.java @@ -47,6 +47,8 @@ import org.openrdf.repository.sail.SailRepositoryConnection; import org.openrdf.sail.Sail; import org.openrdf.sail.SailException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -56,12 +58,13 @@ */ @DefaultAnnotation(NonNull.class) public class MongoExecuteSparqlQuery implements ExecuteSparqlQuery { + private static final Logger log = LoggerFactory.getLogger(MongoExecuteSparqlQuery.class); private final MongoConnectionDetails connectionDetails; private final InstanceExists instanceExists; /** - * Constructs an instance. + * Constructs an instance of {@link MongoExecuteSparqlQuery}. * * @param connectionDetails - Details to connect to the server. (not null) * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) @@ -72,32 +75,25 @@ public MongoExecuteSparqlQuery( this.connectionDetails = requireNonNull(connectionDetails); this.instanceExists = requireNonNull(instanceExists); } + @Override public String executeSparqlQuery(final String ryaInstanceName, final String sparqlQuery) throws InstanceDoesNotExistException, RyaClientException { requireNonNull(ryaInstanceName); requireNonNull(sparqlQuery); - requireNonNull(ryaInstanceName); - requireNonNull(sparqlQuery); // Ensure the Rya Instance exists. if (!instanceExists.exists(ryaInstanceName)) { - throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", - ryaInstanceName)); + throw new InstanceDoesNotExistException(String.format("There is no Rya instance named '%s'.", ryaInstanceName)); } + Sail sail = null; - SailRepository sailRepo = null; SailRepositoryConnection sailRepoConn = null; - // Get a Sail object that is connected to the Rya instance. - final MongoDBRdfConfiguration ryaConf = connectionDetails.build(ryaInstanceName); try { + // Get a Sail object that is connected to the Rya instance. + final MongoDBRdfConfiguration ryaConf = connectionDetails.build(ryaInstanceName); sail = RyaSailFactory.getInstance(ryaConf); - } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException | AccumuloSecurityException e) { - throw new RyaClientException("While getting a sail instance.", e); - } - // Load the file. - sailRepo = new SailRepository(sail); - try { + final SailRepository sailRepo = new SailRepository(sail); sailRepoConn = sailRepo.getConnection(); // Execute the query. @@ -106,26 +102,37 @@ public String executeSparqlQuery(final String ryaInstanceName, final String spar final ByteArrayOutputStream baos = new ByteArrayOutputStream(); final CountingSPARQLResultsCSVWriter handler = new CountingSPARQLResultsCSVWriter(baos); tupleQuery.evaluate(handler); - final StringBuilder sb = new StringBuilder(); - - final String newline = "\n"; - sb.append("Query Result:").append(newline); - sb.append(new String(baos.toByteArray(), StandardCharsets.UTF_8)); - - final String seconds = new DecimalFormat("0.0##").format((System.currentTimeMillis() - start) / 1000.0); - sb.append("Retrieved ").append(handler.getCount()).append(" results in ").append(seconds).append(" seconds."); - - return sb.toString(); - - } catch (final MalformedQueryException e) { - throw new RyaClientException("There was a problem parsing the supplied query.", e); - } catch (final QueryEvaluationException | TupleQueryResultHandlerException e) { - throw new RyaClientException("There was a problem evaluating the supplied query.", e); - } catch (final RepositoryException e) { - throw new RyaClientException("There was a problem executing the query against the Rya instance named " - + ryaInstanceName + ".", e); - } finally { - // close anything? + final long end = System.currentTimeMillis(); + + // Format and return the result of the query. + final String queryResult = new String(baos.toByteArray(), StandardCharsets.UTF_8); + final String queryDuration = new DecimalFormat("0.0##").format((end - start) / 1000.0); + + return "Query Result:\n" + + queryResult + + "Retrieved " + handler.getCount() + " results in " + queryDuration + " seconds."; + + } catch (SailException | RyaDAOException | InferenceEngineException | AccumuloException | AccumuloSecurityException e) { + throw new RyaClientException("Could not create the Sail object used to query the RYA instance.", e); + } catch (final MalformedQueryException | QueryEvaluationException | TupleQueryResultHandlerException | RepositoryException e) { + throw new RyaClientException("Could not execute the SPARQL query.", e); + } finally { + // Close the resources that were opened. + if(sailRepoConn != null) { + try { + sailRepoConn.close(); + } catch (final RepositoryException e) { + log.error("Couldn't close the SailRepositoryConnection object.", e); + } + } + + if(sail != null) { + try { + sail.shutDown(); + } catch (final SailException e) { + log.error("Couldn't close the Sail object.", e); + } + } } } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java index 37f1d95ee..8c9e0b5e7 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoLoadStatements.java @@ -36,6 +36,8 @@ import org.openrdf.repository.sail.SailRepositoryConnection; import org.openrdf.sail.Sail; import org.openrdf.sail.SailException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -45,12 +47,13 @@ */ @DefaultAnnotation(NonNull.class) public class MongoLoadStatements implements LoadStatements { + private static final Logger log = LoggerFactory.getLogger(MongoLoadStatements.class); private final MongoConnectionDetails connectionDetails; private final InstanceExists instanceExists; /** - * Constructs an instance. + * Constructs an instance of {@link MongoLoadStatements}. * * @param connectionDetails - Details to connect to the server. (not null) * @param instanceExists - The interactor used to check if a Rya instance exists. (not null) @@ -71,24 +74,40 @@ public void loadStatements(final String ryaInstanceName, final Iterable loadMe = installAndLoad(); - // Here comes the method to test + + final InstallConfiguration installConfig = InstallConfiguration.builder() + .setEnableTableHashPrefix(false) + .setEnableEntityCentricIndex(false) + .setEnableFreeTextIndex(false) + .setEnableTemporalIndex(false) + .setEnablePcjIndex(false) + .setEnableGeoIndex(false) + .build(); + ryaClient.getInstall().install(conf.getRyaInstanceName(), installConfig); + + // Load some statements into that instance. + final List statements = makeTestStatements(); + ryaClient.getLoadStatements().loadStatements(conf.getRyaInstanceName(), statements); + + // Execute the SPARQL against the Rya instance. final ExecuteSparqlQuery executeSparql = ryaClient.getExecuteSparqlQuery(); final String sparql = "SELECT * where { ?a ?b ?c }"; - final String results = executeSparql.executeSparqlQuery(conf.getMongoDBName(), sparql); - System.out.println(results); + final String results = executeSparql.executeSparqlQuery(conf.getRyaInstanceName(), sparql); + + // Show the result matches what is expected. assertTrue("result has header.", results.startsWith("Query Result:")); assertTrue("result has column headings.", results.contains("a,b,c")); assertTrue("result has footer.", results.contains("Retrieved 3 results in")); - for (final Statement expect : loadMe) { + for (final Statement expect : statements) { assertTrue("All results should contain expected subjects:", results.contains(expect.getSubject().stringValue())); assertTrue("All results should contain expected predicates:", @@ -66,28 +80,6 @@ public void ExecuteSparqlQuery_exec() throws MongoException, DuplicateInstanceNa } } - private List installAndLoad() throws DuplicateInstanceNameException, RyaClientException { - // first install rya - final InstallConfiguration installConfig = InstallConfiguration.builder() - .setEnableTableHashPrefix(false) - .setEnableEntityCentricIndex(false) - .setEnableFreeTextIndex(false) - .setEnableTemporalIndex(false) - .setEnablePcjIndex(false) - .setEnableGeoIndex(false) - .build(); - final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); - final Install install = ryaClient.getInstall(); - install.install(conf.getMongoDBName(), installConfig); - // next, load data - final List loadMe = makeTestStatements(); - ryaClient.getLoadStatements().loadStatements( - conf.getMongoDBName(), - loadMe); - return loadMe; - } - /** * @return some data to load */ diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index 0f84fea1c..674a8068f 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -21,8 +21,8 @@ import static org.junit.Assert.assertEquals; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; +import java.util.HashSet; +import java.util.Set; import org.apache.rya.api.client.Install; import org.apache.rya.api.client.Install.InstallConfiguration; @@ -63,30 +63,36 @@ public void loadTurtleFile() throws Exception { final MongoConnectionDetails connectionDetails = getConnectionDetails(); final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); final Install install = ryaClient.getInstall(); - install.install(conf.getMongoDBName(), installConfig); + install.install(conf.getRyaInstanceName(), installConfig); // Load the test statement file. ryaClient.getLoadStatementsFile().loadStatements( - conf.getMongoDBName(), + conf.getRyaInstanceName(), Paths.get("src/test/resources/example.ttl"), RDFFormat.TURTLE); // Verify that the statements were loaded. final ValueFactory vf = new ValueFactoryImpl(); - final List expected = new ArrayList<>(); + final Set expected = new HashSet<>(); expected.add(vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#bob"))); expected.add(vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf.createURI("http://example#charlie"))); expected.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf.createURI("http://example#icecream"))); - final List statements = new ArrayList<>(); - final MongoCursor x = getRyaCollection().find().iterator(); - while (x.hasNext()) { - final Document y = x.next(); - statements.add(vf.createStatement(vf.createURI(y.getString("subject")), vf.createURI(y.getString("predicate")), vf.createURI(y.getString("object")))); + final Set statements = new HashSet<>(); + final MongoCursor triplesIterator = getMongoClient() + .getDatabase( conf.getRyaInstanceName() ) + .getCollection( conf.getTriplesCollectionName() ) + .find().iterator(); + while (triplesIterator.hasNext()) { + final Document triple = triplesIterator.next(); + statements.add(vf.createStatement( + vf.createURI(triple.getString("subject")), + vf.createURI(triple.getString("predicate")), + vf.createURI(triple.getString("object")))); } - assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); - assertEquals("All rows in DB should match expected rows:", expected, statements); + + assertEquals(expected, statements); } private MongoConnectionDetails getConnectionDetails() { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java index 47b953da0..b0d0e5b95 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java @@ -20,15 +20,12 @@ import static org.junit.Assert.assertEquals; -import java.util.ArrayList; -import java.util.List; +import java.util.HashSet; +import java.util.Set; -import org.apache.rya.api.client.Install; -import org.apache.rya.api.client.Install.DuplicateInstanceNameException; import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.RyaClient; -import org.apache.rya.api.client.RyaClientException; import org.apache.rya.mongodb.MongoTestBase; import org.bson.Document; import org.junit.Test; @@ -43,9 +40,10 @@ */ public class MongoLoadStatementsIT extends MongoTestBase { + private static final ValueFactory VF = new ValueFactoryImpl(); + @Test(expected = InstanceDoesNotExistException.class) public void instanceDoesNotExist() throws Exception { - org.apache.log4j.BasicConfigurator.configure(); final RyaClient ryaClient = MongoRyaClientFactory.build(getConnectionDetails(), getMongoClient()); // Skip the install step to create error causing situation. ryaClient.getLoadStatements().loadStatements(getConnectionDetails().getHostname(), makeTestStatements()); @@ -55,40 +53,11 @@ public void instanceDoesNotExist() throws Exception { * Pass a list of statements to our loadStatement class. */ @Test - public void loadTurtleFile() throws Exception { + public void loadStatements() throws Exception { // Install an instance of Rya. - final List loadMe = installAndLoad(); - final List stmtResults = new ArrayList<>(); - final MongoCursor triplesIterator = getRyaCollection().find().iterator(); - final ValueFactory vf = new ValueFactoryImpl(); - while (triplesIterator.hasNext()) { - final Document triple = triplesIterator.next(); - stmtResults.add(vf.createStatement(vf.createURI(triple.getString("subject")), vf.createURI(triple.getString( - "predicate")), vf.createURI(triple.getString("object")))); - } - stmtResults.sort(((stmt1, stmt2) -> stmt1.getSubject().toString().compareTo(stmt2.getSubject().toString()))); - assertEquals("Expect all rows to be read.", 3, getRyaCollection().count()); - assertEquals("All rows in DB should match expected rows:", loadMe, stmtResults); - } - - /** - * @return some data to load - */ - private List makeTestStatements() { - final List loadMe = new ArrayList<>(); - final ValueFactory vf = new ValueFactoryImpl(); - - loadMe.add(vf.createStatement(vf.createURI("http://example#alice"), vf.createURI("http://example#talksTo"), vf - .createURI("http://example#bob"))); - loadMe.add(vf.createStatement(vf.createURI("http://example#bob"), vf.createURI("http://example#talksTo"), vf - .createURI("http://example#charlie"))); - loadMe.add(vf.createStatement(vf.createURI("http://example#charlie"), vf.createURI("http://example#likes"), vf - .createURI("http://example#icecream"))); - return loadMe; - } + final MongoConnectionDetails connectionDetails = getConnectionDetails(); + final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); - private List installAndLoad() throws DuplicateInstanceNameException, RyaClientException { - // first install rya final InstallConfiguration installConfig = InstallConfiguration.builder() .setEnableTableHashPrefix(false) .setEnableEntityCentricIndex(false) @@ -97,16 +66,50 @@ private List installAndLoad() throws DuplicateInstanceNameException, .setEnablePcjIndex(false) .setEnableGeoIndex(false) .build(); - final MongoConnectionDetails connectionDetails = getConnectionDetails(); - final RyaClient ryaClient = MongoRyaClientFactory.build(connectionDetails, getMongoClient()); - final Install install = ryaClient.getInstall(); - install.install(conf.getMongoDBName(), installConfig); - // next, load data - final List loadMe = makeTestStatements(); - ryaClient.getLoadStatements().loadStatements( - conf.getMongoDBName(), - loadMe); - return loadMe; + ryaClient.getInstall().install(conf.getRyaInstanceName(), installConfig); + + // Create the statements that will be loaded. + final Set statements = makeTestStatements(); + + // Load them. + ryaClient.getLoadStatements().loadStatements(conf.getRyaInstanceName(), statements); + + // Fetch the statements that have been stored in Mongo DB. + final Set stmtResults = new HashSet<>(); + final MongoCursor triplesIterator = getMongoClient() + .getDatabase( conf.getRyaInstanceName() ) + .getCollection( conf.getTriplesCollectionName() ) + .find().iterator(); + + while (triplesIterator.hasNext()) { + final Document triple = triplesIterator.next(); + stmtResults.add(VF.createStatement( + VF.createURI(triple.getString("subject")), + VF.createURI(triple.getString("predicate")), + VF.createURI(triple.getString("object")))); + } + + // Show the discovered statements match the original statements. + assertEquals(statements, stmtResults); + } + + public Set makeTestStatements() { + final Set statements = new HashSet<>(); + statements.add(VF.createStatement( + VF.createURI("http://example#alice"), + VF.createURI("http://example#talksTo"), + VF.createURI("http://example#bob"))); + statements.add( + VF.createStatement( + VF.createURI("http://example#bob"), + VF.createURI("http://example#talksTo"), + VF.createURI("http://example#charlie"))); + statements.add( + VF.createStatement( + VF.createURI("http://example#charlie"), + VF.createURI("http://example#likes"), + VF.createURI("http://example#icecream"))); + return statements; } private MongoConnectionDetails getConnectionDetails() { diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java index 4cb14fa28..c6bdbb318 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java @@ -121,7 +121,7 @@ public String printConnectionDetails() { case MONGO: final MongoConnectionDetails mongoDetails = sharedState.getShellState().getMongoDetails().get(); - StringBuilder message = new StringBuilder() + final StringBuilder message = new StringBuilder() .append("The shell is connected to an instance of MongoDB using the following parameters:\\n") .append(" Hostname: " + mongoDetails.getHostname() + "\n") .append(" Port: " + mongoDetails.getPort() + "\n"); @@ -189,7 +189,7 @@ public String connectToMongo( Optional.ofNullable(password)); // Connect to a MongoDB server. TODO Figure out how to provide auth info? - MongoClient adminClient = new MongoClient(hostname, Integer.parseInt(port)); + final MongoClient adminClient = new MongoClient(hostname, Integer.parseInt(port)); // Make sure the client is closed at shutdown. Runtime.getRuntime().addShutdownHook(new Thread() { @@ -215,14 +215,8 @@ public void connectToInstance( @CliOption(key = {"instance"}, mandatory = true, help = "The name of the Rya instance the shell will interact with.") final String instance) { try { - // TODO When you are connected to mongo db, then connecting to an instance may require - // a username/password. this is because each Database has its own credentials, so - // every rya instance likewise has their own credentials. - final InstanceExists instanceExists = sharedState.getShellState().getConnectedCommands().get().getInstanceExists(); - // TODO gracefully fail if that version doen't support it. maybe the list command should go ahead - // Make sure the requested instance exists. if(!instanceExists.exists(instance)) { throw new RuntimeException(String.format("'%s' does not match an existing Rya instance.", instance)); @@ -237,6 +231,13 @@ public void connectToInstance( @CliCommand(value = DISCONNECT_COMMAND_NAME_CMD, help = "Disconnect the shell's Rya storage connection (Accumulo).") public void disconnect() { + // If connected to Mongo, there is a client that needs to be closed. + final com.google.common.base.Optional mongoAdminClient = sharedState.getShellState().getMongoAdminClient(); + if(mongoAdminClient.isPresent()) { + mongoAdminClient.get().close(); + } + + // Update the shared state to disconnected. sharedState.disconnected(); } } \ No newline at end of file diff --git a/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java b/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java index 89cbd5d95..bb22fd3cc 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/SharedShellState.java @@ -28,6 +28,7 @@ import org.apache.rya.api.client.mongo.MongoConnectionDetails; import com.google.common.base.Optional; +import com.mongodb.MongoClient; import edu.umd.cs.findbugs.annotations.DefaultAnnotation; import edu.umd.cs.findbugs.annotations.NonNull; @@ -222,6 +223,7 @@ public static final class ShellState { private final Optional storageType; private final Optional accumuloDetails; private final Optional mongoDetails; + private final Optional mongoAdminClient; private final Optional connectedCommands; // Instance specific values. @@ -232,12 +234,14 @@ private ShellState( final Optional storageType, final Optional accumuloDetails, final Optional mongoDetails, + final Optional mongoAdminClient, final Optional connectedCommands, final Optional instanceName) { this.connectionState = requireNonNull(connectionState); this.storageType = requireNonNull(storageType); this.accumuloDetails = requireNonNull(accumuloDetails); this.mongoDetails = requireNonNull(mongoDetails); + this.mongoAdminClient = requireNonNull(mongoAdminClient); this.connectedCommands = requireNonNull(connectedCommands); this.instanceName = requireNonNull(instanceName); } @@ -274,6 +278,13 @@ public Optional getMongoDetails() { return mongoDetails; } + /** + * @return The Mongo Client that is used to perform administrative tasks while connected to Mongo DB. + */ + public Optional getMongoAdminClient() { + return mongoAdminClient; + } + /** * @return The {@link RyaClient} to use when a command on the shell is issued. * The value will not be present if the Rya Shell is not connected to a storage. @@ -340,6 +351,7 @@ public static class Builder { private StorageType storageType; private AccumuloConnectionDetails accumuloDetails; private MongoConnectionDetails mongoDetails; + private MongoClient mongoAdminClient; private RyaClient connectedCommands; // Instance specific values. @@ -361,6 +373,7 @@ public Builder(final ShellState shellState) { this.storageType = shellState.getStorageType().orNull(); this.accumuloDetails = shellState.getAccumuloDetails().orNull(); this.mongoDetails = shellState.getMongoDetails().orNull(); + this.mongoAdminClient = shellState.getMongoAdminClient().orNull(); this.connectedCommands = shellState.getConnectedCommands().orNull(); this.instanceName = shellState.getRyaInstanceName().orNull(); } @@ -402,6 +415,16 @@ public Builder setMongoDetails(@Nullable final MongoConnectionDetails mongoDetai return this; } + /** + * @param mongoAdminClient - The Mongo Client that is used to perform administrative tasks while + * connected to Mongo DB. + * @return This {@link Builder} so that method invocations may be chained. + */ + public Builder setMongoAdminClient(@Nullable final MongoClient mongoAdminClient) { + this.mongoAdminClient = mongoAdminClient; + return this; + } + /** * @param connectedCommands - The {@link RyaClient} to use when a command on the shell is issued. * @return This {@link Builder} so that method invocations may be chained. @@ -429,6 +452,7 @@ public ShellState build() { Optional.fromNullable(storageType), Optional.fromNullable(accumuloDetails), Optional.fromNullable(mongoDetails), + Optional.fromNullable(mongoAdminClient), Optional.fromNullable(connectedCommands), Optional.fromNullable(instanceName)); } From 913d7b0a351fd7221446d7ea503890f2bbcbd83e Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Fri, 29 Dec 2017 13:27:27 -0500 Subject: [PATCH 10/13] RYA-104 Updated the Rya Shell to prompt for Mongo Specific install configurations. --- .../apache/rya/shell/util/InstallPrompt.java | 131 ++++++++++++++++-- .../META-INF/spring/spring-shell-plugin.xml | 3 +- 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java b/extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java index 5d9d48b6b..e92eaddf4 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/util/InstallPrompt.java @@ -18,12 +18,20 @@ */ package org.apache.rya.shell.util; +import static com.google.common.base.Preconditions.checkState; +import static java.util.Objects.requireNonNull; + import java.io.IOException; import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.shell.SharedShellState; +import org.apache.rya.shell.SharedShellState.StorageType; +import org.springframework.beans.factory.annotation.Autowired; import com.google.common.base.Optional; +import edu.umd.cs.findbugs.annotations.DefaultAnnotation; +import edu.umd.cs.findbugs.annotations.NonNull; import jline.console.ConsoleReader; /** @@ -43,7 +51,7 @@ public interface InstallPrompt { /** * Prompt the user for which features of Rya they want enabled. * - * @param instanceName - The Rya instance name. + * @param instanceName - The Rya instance name. (not null) * @return The value they entered. * @throws IOException There was a problem reading the values. */ @@ -53,6 +61,8 @@ public interface InstallPrompt { * Prompt the user asking them if they are sure they would like to do the * install. * + * @param instanceName - The Rya instance name. (not null) + * @param installConfig - The configuration that will be presented to the user. (not null) * @return The value they entered. * @throws IOException There was a problem reading the value. */ @@ -60,8 +70,13 @@ public interface InstallPrompt { /** * Prompts a user for install information using a JLine {@link ConsoleReader}. + * The prompt it uses depends on the storage that is connected to. */ - public static class JLineAccumuloInstallPrompt extends JLinePrompt implements InstallPrompt { + @DefaultAnnotation(NonNull.class) + public static class JLineInstallPropmpt extends JLinePrompt implements InstallPrompt { + + @Autowired + private SharedShellState sharedShellState; @Override public String promptInstanceName() throws IOException { @@ -72,6 +87,48 @@ public String promptInstanceName() throws IOException { @Override public InstallConfiguration promptInstallConfiguration(final String instanceName) throws IOException { + final Optional storageType = sharedShellState.getShellState().getStorageType(); + checkState(storageType.isPresent(), "The shell must be connected to a storage to use the install prompt."); + + switch(sharedShellState.getShellState().getStorageType().get()) { + case ACCUMULO: + return promptAccumuloConfig(instanceName); + + case MONGO: + return promptMongoConfig(instanceName); + + default: + throw new IllegalStateException("Unsupported storage type: " + storageType.get()); + } + } + + @Override + public boolean promptVerified(final String instanceName, final InstallConfiguration installConfig) throws IOException { + final Optional storageType = sharedShellState.getShellState().getStorageType(); + checkState(storageType.isPresent(), "The shell must be connected to a storage to use the install prompt."); + + switch(sharedShellState.getShellState().getStorageType().get()) { + case ACCUMULO: + return promptAccumuloVerified(instanceName, installConfig); + + case MONGO: + return promptMongoVerified(instanceName, installConfig); + + default: + throw new IllegalStateException("Unsupported storage type: " + storageType.get()); + } + } + + /** + * Prompt the user for which Accumulo specific features of Rya they want enabled. + * + * @param instanceName - The Rya instance name. (not null) + * @return The value they entered. + * @throws IOException There was a problem reading the values. + */ + private InstallConfiguration promptAccumuloConfig(final String instanceName) throws IOException { + requireNonNull(instanceName); + final InstallConfiguration.Builder builder = InstallConfiguration.builder(); String prompt = makeFieldPrompt("Use Shard Balancing (improves streamed input write speeds)", false); @@ -86,9 +143,9 @@ public InstallConfiguration promptInstallConfiguration(final String instanceName final boolean enableFreeTextIndexing = promptBoolean(prompt, Optional.of(true)); builder.setEnableFreeTextIndex( enableFreeTextIndexing ); - prompt = makeFieldPrompt("Use Geospatial Indexing", true); - final boolean enableGeoIndexing = promptBoolean(prompt, Optional.of(true)); - builder.setEnableGeoIndex( enableGeoIndexing ); +// RYA-215 prompt = makeFieldPrompt("Use Geospatial Indexing", true); +// final boolean enableGeoIndexing = promptBoolean(prompt, Optional.of(true)); +// builder.setEnableGeoIndex( enableGeoIndexing ); prompt = makeFieldPrompt("Use Temporal Indexing", true); final boolean useTemporalIndexing = promptBoolean(prompt, Optional.of(true)); @@ -111,8 +168,19 @@ public InstallConfiguration promptInstallConfiguration(final String instanceName return builder.build(); } - @Override - public boolean promptVerified(final String instanceName, final InstallConfiguration installConfig) throws IOException { + /** + * Prompt the user asking them if they are sure they would like to do the + * install. + * + * @param instanceName - The Rya instance name. (not null) + * @param installConfig - The configuration that will be presented to the user. (not null) + * @return The value they entered. + * @throws IOException There was a problem reading the value. + */ + private boolean promptAccumuloVerified(final String instanceName, final InstallConfiguration installConfig) throws IOException { + requireNonNull(instanceName); + requireNonNull(installConfig); + final ConsoleReader reader = getReader(); reader.println(); reader.println("A Rya instance will be installed using the following values:"); @@ -120,7 +188,7 @@ public boolean promptVerified(final String instanceName, final InstallConfigurat reader.println(" Use Shard Balancing: " + installConfig.isTableHashPrefixEnabled()); reader.println(" Use Entity Centric Indexing: " + installConfig.isEntityCentrixIndexEnabled()); reader.println(" Use Free Text Indexing: " + installConfig.isFreeTextIndexEnabled()); - reader.println(" Use Geospatial Indexing: " + installConfig.isGeoIndexEnabled()); +// RYA-215 reader.println(" Use Geospatial Indexing: " + installConfig.isGeoIndexEnabled()); reader.println(" Use Temporal Indexing: " + installConfig.isTemporalIndexEnabled()); reader.println(" Use Precomputed Join Indexing: " + installConfig.isPcjIndexEnabled()); if(installConfig.isPcjIndexEnabled()) { @@ -135,5 +203,52 @@ public boolean promptVerified(final String instanceName, final InstallConfigurat return promptBoolean("Continue with the install? (y/n) ", Optional.absent()); } + + /** + * Prompt the user for which Mongo specific features of Rya they want enabled. + * + * @param instanceName - The Rya instance name. (not null) + * @return The value they entered. + * @throws IOException There was a problem reading the values. + */ + private InstallConfiguration promptMongoConfig(final String instanceName) throws IOException { + requireNonNull(instanceName); + + final InstallConfiguration.Builder builder = InstallConfiguration.builder(); + + String prompt = makeFieldPrompt("Use Free Text Indexing", true); + final boolean enableFreeTextIndexing = promptBoolean(prompt, Optional.of(true)); + builder.setEnableFreeTextIndex( enableFreeTextIndexing ); + + prompt = makeFieldPrompt("Use Temporal Indexing", true); + final boolean useTemporalIndexing = promptBoolean(prompt, Optional.of(true)); + builder.setEnableTemporalIndex( useTemporalIndexing ); + + return builder.build(); + } + + /** + * Prompt the user asking them if they are sure they would like to do the + * install. + * + * @param instanceName - The Rya instance name. (not null) + * @param installConfig - The configuration that will be presented to the user. (not null) + * @return The value they entered. + * @throws IOException There was a problem reading the value. + */ + private boolean promptMongoVerified(final String instanceName, final InstallConfiguration installConfig) throws IOException { + requireNonNull(instanceName); + requireNonNull(installConfig); + + final ConsoleReader reader = getReader(); + reader.println(); + reader.println("A Rya instance will be installed using the following values:"); + reader.println(" Instance Name: " + instanceName); + reader.println(" Use Free Text Indexing: " + installConfig.isFreeTextIndexEnabled()); + reader.println(" Use Temporal Indexing: " + installConfig.isTemporalIndexEnabled()); + reader.println(""); + + return promptBoolean("Continue with the install? (y/n) ", Optional.absent()); + } } } \ No newline at end of file diff --git a/extras/shell/src/main/resources/META-INF/spring/spring-shell-plugin.xml b/extras/shell/src/main/resources/META-INF/spring/spring-shell-plugin.xml index 48c48467c..361bf2753 100644 --- a/extras/shell/src/main/resources/META-INF/spring/spring-shell-plugin.xml +++ b/extras/shell/src/main/resources/META-INF/spring/spring-shell-plugin.xml @@ -31,7 +31,8 @@ - + + From 00800b68e9eccfaf9303e7a750c862907593c820 Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Fri, 29 Dec 2017 17:56:48 -0500 Subject: [PATCH 11/13] RYA-104 Integration tests with Mongo for the shell. --- .../client/mongo/MongoConnectionDetails.java | 3 +- .../mongo/MongoExecuteSparqlQueryIT.java | 4 +- .../mongo/MongoGetInstanceDetailsIT.java | 4 +- .../rya/api/client/mongo/MongoInstallIT.java | 4 +- .../client/mongo/MongoInstanceExistsIT.java | 4 +- .../client/mongo/MongoListInstancesIT.java | 4 +- .../mongo/MongoLoadStatementsFileIT.java | 4 +- .../client/mongo/MongoLoadStatementsIT.java | 4 +- .../api/client/mongo/MongoUninstallIT.java | 4 +- extras/shell/pom.xml | 6 + .../rya/shell/RyaConnectionCommands.java | 14 +- ...a => AccumuloRyaConnectionCommandsIT.java} | 73 +------ .../org/apache/rya/shell/MongoRyaShellIT.java | 194 ++++++++++++++++++ ...TBase.java => RyaShellAccumuloITBase.java} | 2 +- .../apache/rya/shell/RyaShellMongoITBase.java | 71 +++++++ 15 files changed, 297 insertions(+), 98 deletions(-) rename extras/shell/src/test/java/org/apache/rya/shell/{RyaConnectionCommandsIT.java => AccumuloRyaConnectionCommandsIT.java} (76%) create mode 100644 extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java rename extras/shell/src/test/java/org/apache/rya/shell/{RyaShellITBase.java => RyaShellAccumuloITBase.java} (98%) create mode 100644 extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index b48bb9f99..acce15f21 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -112,8 +112,7 @@ public MongoDBRdfConfiguration build(final String ryaInstanceName) { conf.setMongoPassword( new String(password.get()) ); } - // Both of these are ways to configure the collection prefixes. - //TODO these should not be explicitly set + // Default to having a table prefix of "rya", otherwise the app doesn't work for some reason. conf.setTablePrefix("rya"); return conf; } diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java index 1d38410f0..5c1411273 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoExecuteSparqlQueryIT.java @@ -28,7 +28,7 @@ import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.junit.Test; import org.openrdf.model.Statement; import org.openrdf.model.ValueFactory; @@ -39,7 +39,7 @@ /** * Integration tests the methods of {@link }. */ -public class MongoExecuteSparqlQueryIT extends MongoTestBase { +public class MongoExecuteSparqlQueryIT extends MongoITBase { @Test public void ExecuteSparqlQuery_exec() throws MongoException, DuplicateInstanceNameException, RyaClientException { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java index e68ec49fd..b3d96f6b7 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoGetInstanceDetailsIT.java @@ -37,7 +37,7 @@ import org.apache.rya.api.instance.RyaDetails.PCJIndexDetails; import org.apache.rya.api.instance.RyaDetails.ProspectorDetails; import org.apache.rya.api.instance.RyaDetails.TemporalIndexDetails; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.junit.Test; import com.google.common.base.Optional; @@ -46,7 +46,7 @@ /** * Tests the methods of {@link MongoGetInstanceDetails}. */ -public class MongoGetInstanceDetailsIT extends MongoTestBase { +public class MongoGetInstanceDetailsIT extends MongoITBase { @Test public void getDetails() throws MongoException, RyaClientException { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java index 16991c766..e3df91662 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstallIT.java @@ -33,13 +33,13 @@ import org.apache.rya.api.client.InstanceExists; import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.junit.Test; /** * Integration tests the methods of {@link MongoInstall}. */ -public class MongoInstallIT extends MongoTestBase { +public class MongoInstallIT extends MongoITBase { @Test public void install() throws DuplicateInstanceNameException, RyaClientException { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java index a2e8e25ab..c27df2c1c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoInstanceExistsIT.java @@ -22,7 +22,7 @@ import static org.junit.Assert.assertTrue; import org.apache.accumulo.core.client.TableExistsException; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.apache.rya.mongodb.instance.MongoRyaInstanceDetailsRepository; import org.junit.Test; @@ -32,7 +32,7 @@ /** * Integration tests the methods of {@link MongoInstanceExistsExists}. */ -public class MongoInstanceExistsIT extends MongoTestBase { +public class MongoInstanceExistsIT extends MongoITBase { @Test public void exists_ryaDetailsTable() throws MongoException, TableExistsException { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java index c66670e8f..6a5b1d66c 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoListInstancesIT.java @@ -30,7 +30,7 @@ import org.apache.rya.api.client.ListInstances; import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.junit.Test; import com.google.common.collect.Lists; @@ -39,7 +39,7 @@ /** * Integration tests the methods of {@link MongoListInstances}. */ -public class MongoListInstancesIT extends MongoTestBase { +public class MongoListInstancesIT extends MongoITBase { @Test public void listInstances_hasRyaDetailsTable() throws MongoException, DuplicateInstanceNameException, RyaClientException { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java index 674a8068f..2ecaec0a7 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsFileIT.java @@ -28,7 +28,7 @@ import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.RyaClient; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.bson.Document; import org.junit.Test; import org.openrdf.model.Statement; @@ -40,7 +40,7 @@ /** * Integration tests the methods of {@link MongoLoadStatementsFile}. */ -public class MongoLoadStatementsFileIT extends MongoTestBase { +public class MongoLoadStatementsFileIT extends MongoITBase { @Test(expected = InstanceDoesNotExistException.class) public void instanceDoesNotExist() throws Exception { diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java index b0d0e5b95..57e4b766f 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoLoadStatementsIT.java @@ -26,7 +26,7 @@ import org.apache.rya.api.client.Install.InstallConfiguration; import org.apache.rya.api.client.InstanceDoesNotExistException; import org.apache.rya.api.client.RyaClient; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.bson.Document; import org.junit.Test; import org.openrdf.model.Statement; @@ -38,7 +38,7 @@ /** * Integration tests the methods of {@link MongoLoadStatements}. */ -public class MongoLoadStatementsIT extends MongoTestBase { +public class MongoLoadStatementsIT extends MongoITBase { private static final ValueFactory VF = new ValueFactoryImpl(); diff --git a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java index 6c9d4fd29..d9d1a6d19 100644 --- a/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java +++ b/extras/indexing/src/test/java/org/apache/rya/api/client/mongo/MongoUninstallIT.java @@ -30,7 +30,7 @@ import org.apache.rya.api.client.RyaClient; import org.apache.rya.api.client.RyaClientException; import org.apache.rya.api.client.Uninstall; -import org.apache.rya.mongodb.MongoTestBase; +import org.apache.rya.mongodb.MongoITBase; import org.junit.Test; import com.mongodb.MongoException; @@ -38,7 +38,7 @@ /** * Integration tests the methods of {@link MongoUninstall}. */ -public class MongoUninstallIT extends MongoTestBase { +public class MongoUninstallIT extends MongoITBase { @Test public void uninstall() throws MongoException, RyaClientException { diff --git a/extras/shell/pom.xml b/extras/shell/pom.xml index a2aedbc12..1a074006a 100644 --- a/extras/shell/pom.xml +++ b/extras/shell/pom.xml @@ -99,6 +99,12 @@ test-jar test + + org.apache.rya + mongodb.rya + test-jar + test + diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java index c6bdbb318..d5b71172f 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaConnectionCommands.java @@ -88,11 +88,11 @@ public boolean areConnectCommandsAvailable() { @CliAvailabilityIndicator({CONNECT_INSTANCE_CMD}) public boolean isConnectToInstanceAvailable() { switch(sharedState.getShellState().getConnectionState()) { - case CONNECTED_TO_STORAGE: - case CONNECTED_TO_INSTANCE: - return true; - default: - return false; + case CONNECTED_TO_STORAGE: + case CONNECTED_TO_INSTANCE: + return true; + default: + return false; } } @@ -122,12 +122,12 @@ public String printConnectionDetails() { final MongoConnectionDetails mongoDetails = sharedState.getShellState().getMongoDetails().get(); final StringBuilder message = new StringBuilder() - .append("The shell is connected to an instance of MongoDB using the following parameters:\\n") + .append("The shell is connected to an instance of MongoDB using the following parameters:\n") .append(" Hostname: " + mongoDetails.getHostname() + "\n") .append(" Port: " + mongoDetails.getPort() + "\n"); if(mongoDetails.getUsername().isPresent()) { - message.append(" Username:" + mongoDetails.getUsername().get()); + message.append(" Username: " + mongoDetails.getUsername().get() + "\n"); } return message.toString(); diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaConnectionCommandsIT.java b/extras/shell/src/test/java/org/apache/rya/shell/AccumuloRyaConnectionCommandsIT.java similarity index 76% rename from extras/shell/src/test/java/org/apache/rya/shell/RyaConnectionCommandsIT.java rename to extras/shell/src/test/java/org/apache/rya/shell/AccumuloRyaConnectionCommandsIT.java index 45bfced3e..98fb64cf1 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaConnectionCommandsIT.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/AccumuloRyaConnectionCommandsIT.java @@ -40,7 +40,7 @@ /** * Integration tests the methods of {@link RyaConnectionCommands}. */ -public class RyaConnectionCommandsIT extends RyaShellITBase { +public class AccumuloRyaConnectionCommandsIT extends RyaShellAccumuloITBase { @Test public void connectAccumulo() throws IOException { @@ -66,54 +66,6 @@ public void connectAccumulo() throws IOException { assertTrue( connectResult.isSuccess() ); } - @Test - public void connectMongo() throws IOException { - // FIXME create mongo client/server here - final Bootstrap bootstrap = getTestBootstrap(); - final JLineShellComponent shell = getTestShell(); - - // Mock the user entering the correct password. - final ApplicationContext context = bootstrap.getApplicationContext(); - final PasswordPrompt mockPrompt = context.getBean(PasswordPrompt.class); - when(mockPrompt.getPassword()).thenReturn("password".toCharArray()); - - // Execute the connect command. - final String cmd = - RyaConnectionCommands.CONNECT_MONGO_CMD + " " + - "--username root " + - "--hostname " + "localhost" + " " + - "--port " + "999"; - - final CommandResult connectResult = shell.executeCommand(cmd); - - // Ensure the connection was successful. - assertTrue(connectResult.isSuccess()); - } - - @Test - public void connectAccumulo_noAuths() throws IOException { - final MiniAccumuloCluster cluster = getCluster(); - final Bootstrap bootstrap = getTestBootstrap(); - final JLineShellComponent shell = getTestShell(); - - // Mock the user entering the correct password. - final ApplicationContext context = bootstrap.getApplicationContext(); - final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class ); - when(mockPrompt.getPassword()).thenReturn("password".toCharArray()); - - // Execute the command - final String cmd = - RyaConnectionCommands.CONNECT_ACCUMULO_CMD + " " + - "--username root " + - "--instanceName " + cluster.getInstanceName() + " "+ - "--zookeepers " + cluster.getZooKeepers(); - - final CommandResult connectResult = shell.executeCommand(cmd); - - // Ensure the connection was successful. - assertTrue( connectResult.isSuccess() ); - } - @Test public void connectAccumulo_wrongCredentials() throws IOException { final MiniAccumuloCluster cluster = getCluster(); @@ -271,27 +223,4 @@ public void disconnect() throws IOException { final CommandResult disconnectResult = shell.executeCommand( RyaConnectionCommands.DISCONNECT_COMMAND_NAME_CMD ); assertTrue( disconnectResult.isSuccess() ); } - - @Test - public void connectMongo_noAuths() throws IOException { - // FIXME make Mongo connection here - final Bootstrap bootstrap = getTestBootstrap(); - final JLineShellComponent shell = getTestShell(); - - // Mock the user entering the correct password. - final ApplicationContext context = bootstrap.getApplicationContext(); - final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class ); - when(mockPrompt.getPassword()).thenReturn("password".toCharArray()); - - // Execute the command - final String cmd = RyaConnectionCommands.CONNECT_MONGO_CMD + " " + - "--username root " + - "--hostname " + "localhost" + " " + - "--port " + "999"; // FIXME - - final CommandResult connectResult = shell.executeCommand(cmd); - - // Ensure the connection was successful. - assertTrue( connectResult.isSuccess() ); - } } \ No newline at end of file diff --git a/extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java b/extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java new file mode 100644 index 000000000..79fe95de2 --- /dev/null +++ b/extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java @@ -0,0 +1,194 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.shell; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.when; + +import java.io.IOException; + +import org.apache.rya.api.client.Install.InstallConfiguration; +import org.apache.rya.shell.SharedShellState.ConnectionState; +import org.apache.rya.shell.SharedShellState.ShellState; +import org.apache.rya.shell.util.InstallPrompt; +import org.apache.rya.shell.util.PasswordPrompt; +import org.junit.Test; +import org.springframework.context.ApplicationContext; +import org.springframework.shell.Bootstrap; +import org.springframework.shell.core.CommandResult; +import org.springframework.shell.core.JLineShellComponent; + +/** + * Integration tests the functions of the Mongo Rya Shell. + */ +public class MongoRyaShellIT extends RyaShellMongoITBase { + + @Test + public void connectMongo_noAuth() throws IOException { + final JLineShellComponent shell = getTestShell(); + + // Connect to the Mongo instance. + final String cmd = + RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--hostname " + super.conf.getMongoHostname() + " " + + "--port " + super.conf.getMongoPort(); + + final CommandResult connectResult = shell.executeCommand(cmd); + + // Ensure the connection was successful. + assertTrue(connectResult.isSuccess()); + } + + @Test + public void printConnectionDetails_notConnected() { + final JLineShellComponent shell = getTestShell(); + + // Run the print connection details command. + final CommandResult printResult = shell.executeCommand( RyaConnectionCommands.PRINT_CONNECTION_DETAILS_CMD ); + final String msg = (String) printResult.getResult(); + + final String expected = "The shell is not connected to anything."; + assertEquals(expected, msg); + } + + @Test + public void printConnectionDetails_connectedToMongo_noAuths() throws IOException { + final JLineShellComponent shell = getTestShell(); + + // Connect to the Mongo instance. + final String cmd = + RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--hostname " + super.conf.getMongoHostname() + " " + + "--port " + super.conf.getMongoPort(); + shell.executeCommand(cmd); + + // Run the print connection details command. + final CommandResult printResult = shell.executeCommand( RyaConnectionCommands.PRINT_CONNECTION_DETAILS_CMD ); + final String msg = (String) printResult.getResult(); + + final String expected = + "The shell is connected to an instance of MongoDB using the following parameters:\n" + + " Hostname: " + super.conf.getMongoHostname() + "\n" + + " Port: " + super.conf.getMongoPort() + "\n"; + assertEquals(expected, msg); + } + + @Test + public void printConnectionDetails_connectedToMongo_auths() throws IOException { + final Bootstrap bootstrap = getTestBootstrap(); + final JLineShellComponent shell = getTestShell(); + + // Mock the user entering the correct password. + final ApplicationContext context = bootstrap.getApplicationContext(); + final PasswordPrompt mockPrompt = context.getBean( PasswordPrompt.class ); + when(mockPrompt.getPassword()).thenReturn("password".toCharArray()); + + // Connect to the Mongo instance. + final String cmd = + RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--hostname " + super.conf.getMongoHostname() + " " + + "--port " + super.conf.getMongoPort() + " " + + "--username bob"; + shell.executeCommand(cmd); + + // Run the print connection details command. + final CommandResult printResult = shell.executeCommand( RyaConnectionCommands.PRINT_CONNECTION_DETAILS_CMD ); + final String msg = (String) printResult.getResult(); + + final String expected = + "The shell is connected to an instance of MongoDB using the following parameters:\n" + + " Hostname: " + super.conf.getMongoHostname() + "\n" + + " Port: " + super.conf.getMongoPort() + "\n" + + " Username: bob\n"; + assertEquals(expected, msg); + } + + @Test + public void connectToInstance_instanceDoesNotExist() throws IOException { + final JLineShellComponent shell = getTestShell(); + + // Connect to the Mongo instance. + String cmd = + RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--hostname " + super.conf.getMongoHostname() + " " + + "--port " + super.conf.getMongoPort(); + shell.executeCommand(cmd); + + // Try to connect to a non-existing instance. + cmd = RyaConnectionCommands.CONNECT_INSTANCE_CMD + " --instance doesNotExist"; + final CommandResult result = shell.executeCommand(cmd); + assertFalse( result.isSuccess() ); + } + + @Test + public void connectToInstance_noAuths() throws IOException { + final Bootstrap bootstrap = getTestBootstrap(); + final JLineShellComponent shell = getTestShell(); + + // Connect to the Mongo instance. + String cmd = + RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--hostname " + super.conf.getMongoHostname() + " " + + "--port " + super.conf.getMongoPort(); + shell.executeCommand(cmd); + + // Install an instance of rya. + final String instanceName = "testInstance"; + final InstallConfiguration installConf = InstallConfiguration.builder().build(); + + final ApplicationContext context = bootstrap.getApplicationContext(); + final InstallPrompt installPrompt = context.getBean( InstallPrompt.class ); + when(installPrompt.promptInstanceName()).thenReturn("testInstance"); + when(installPrompt.promptInstallConfiguration("testInstance")).thenReturn( installConf ); + when(installPrompt.promptVerified(instanceName, installConf)).thenReturn(true); + + CommandResult result = shell.executeCommand( RyaAdminCommands.INSTALL_CMD ); + assertTrue( result.isSuccess() ); + + // Connect to the instance that was just installed. + cmd = RyaConnectionCommands.CONNECT_INSTANCE_CMD + " --instance " + instanceName; + result = shell.executeCommand(cmd); + assertTrue( result.isSuccess() ); + + // Verify the shell state indicates it is connected to an instance. + final SharedShellState sharedState = context.getBean( SharedShellState.class ); + final ShellState state = sharedState.getShellState(); + assertEquals(ConnectionState.CONNECTED_TO_INSTANCE, state.getConnectionState()); + } + + @Test + public void disconnect() throws IOException { + final JLineShellComponent shell = getTestShell(); + + // Connect to the Mongo instance. + final String cmd = + RyaConnectionCommands.CONNECT_MONGO_CMD + " " + + "--hostname " + super.conf.getMongoHostname() + " " + + "--port " + super.conf.getMongoPort(); + shell.executeCommand(cmd); + + // Disconnect from it. + final CommandResult disconnectResult = shell.executeCommand( RyaConnectionCommands.DISCONNECT_COMMAND_NAME_CMD ); + assertTrue( disconnectResult.isSuccess() ); + } + + // TODO the rest of them? +} \ No newline at end of file diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaShellITBase.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaShellAccumuloITBase.java similarity index 98% rename from extras/shell/src/test/java/org/apache/rya/shell/RyaShellITBase.java rename to extras/shell/src/test/java/org/apache/rya/shell/RyaShellAccumuloITBase.java index bcbaa5b21..a115639be 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaShellITBase.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaShellAccumuloITBase.java @@ -42,7 +42,7 @@ * and shutdown hooks for a Mini Accumulo Cluster when you start and stop testing. * It also creates a new shell to test with between each test. */ -public class RyaShellITBase { +public class RyaShellAccumuloITBase { /** * The bootstrap that was used to initialize the Shell that will be tested. diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java new file mode 100644 index 000000000..7e83dbf5d --- /dev/null +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.shell; + +import java.io.IOException; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.rya.mongodb.MongoITBase; +import org.junit.After; +import org.junit.Before; +import org.springframework.shell.Bootstrap; +import org.springframework.shell.core.JLineShellComponent; + +/** + * All Rya Shell integration tests should extend this one if they are testing against Mongo DB. + */ +public class RyaShellMongoITBase extends MongoITBase { + + /** + * The bootstrap that was used to initialize the Shell that will be tested. + */ + private Bootstrap bootstrap; + + /** + * The shell that will be tested. + */ + private JLineShellComponent shell; + + @Before + public void startShell() throws IOException, InterruptedException, AccumuloException, AccumuloSecurityException { + // Bootstrap the shell with the test bean configuration. + bootstrap = new Bootstrap(new String[]{}, new String[]{"file:src/test/resources/RyaShellTest-context.xml"}); + shell = bootstrap.getJLineShellComponent(); + } + + @After + public void stopShell() throws IOException, InterruptedException { + shell.stop(); + } + + /** + * @return The bootstrap that was used to initialize the Shell that will be tested. + */ + public Bootstrap getTestBootstrap() { + return bootstrap; + } + + /** + * @return The shell that will be tested. + */ + public JLineShellComponent getTestShell() { + return shell; + } +} \ No newline at end of file From ed82a6ff4031259682dfa73b32923d1cf0a8f589 Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Fri, 5 Jan 2018 16:58:23 -0500 Subject: [PATCH 12/13] RYA-104 Code review. --- .../src/main/java/org/apache/rya/api/client/RyaClient.java | 6 +++--- .../apache/rya/api/client/mongo/MongoConnectionDetails.java | 2 +- .../org/apache/rya/api/client/mongo/MongoUninstall.java | 1 + extras/rya.pcj.fluo/pom.xml | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java b/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java index de0b542ff..a14e30017 100644 --- a/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java +++ b/common/rya.api/src/main/java/org/apache/rya/api/client/RyaClient.java @@ -83,9 +83,9 @@ public RyaClient( this.addUser = requireNonNull(addUser); this.removeUser = requireNonNull(removeUser); this.uninstall = requireNonNull(uninstall); - this.loadStatements = loadStatements; - this.loadStatementsFile = loadStatementsFile; - this.executeSparqlQuery = executeSparqlQuery; + this.loadStatements = requireNonNull(loadStatements); + this.loadStatementsFile = requireNonNull(loadStatementsFile); + this.executeSparqlQuery = requireNonNull(executeSparqlQuery); } /** diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java index acce15f21..192ea2089 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoConnectionDetails.java @@ -112,7 +112,7 @@ public MongoDBRdfConfiguration build(final String ryaInstanceName) { conf.setMongoPassword( new String(password.get()) ); } - // Default to having a table prefix of "rya", otherwise the app doesn't work for some reason. + // Default to having a table prefix of "rya", otherwise the app doesn't work. Remove this in RYA-135. conf.setTablePrefix("rya"); return conf; } diff --git a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java index a6fbb4c10..c45007790 100644 --- a/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java +++ b/extras/indexing/src/main/java/org/apache/rya/api/client/mongo/MongoUninstall.java @@ -53,6 +53,7 @@ public MongoUninstall(final MongoClient adminClient, final MongoInstanceExists i @Override public void uninstall(final String ryaInstanceName) throws InstanceDoesNotExistException, RyaClientException { + requireNonNull(ryaInstanceName); try { if (!instanceExists.exists(ryaInstanceName)) { throw new InstanceDoesNotExistException("The database '" + ryaInstanceName + "' does not exist."); diff --git a/extras/rya.pcj.fluo/pom.xml b/extras/rya.pcj.fluo/pom.xml index 3cae897b2..b6f54033a 100644 --- a/extras/rya.pcj.fluo/pom.xml +++ b/extras/rya.pcj.fluo/pom.xml @@ -37,7 +37,7 @@ pcj.fluo.api pcj.fluo.app pcj.fluo.client - + pcj.fluo.integration pcj.fluo.test.base pcj.fluo.demo From eadfcdd36ef07d76dc4e372a6a3e208c30ff7a53 Mon Sep 17 00:00:00 2001 From: kchilton2 Date: Fri, 5 Jan 2018 15:05:07 -0500 Subject: [PATCH 13/13] RYA-421 Updated the Rya Shell to print a different message when there are no Rya instances to list. --- .../main/java/org/apache/rya/shell/RyaAdminCommands.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java b/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java index cfe13dd6e..f8fee103a 100644 --- a/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java +++ b/extras/shell/src/main/java/org/apache/rya/shell/RyaAdminCommands.java @@ -198,10 +198,17 @@ public String listInstances() { final Optional ryaInstance = shellState.getRyaInstanceName(); try { - // Sort the names alphabetically. final List instanceNames = commands.getListInstances().listInstances(); + + // Return a special message when there are no instances installed. + if(instanceNames.isEmpty()) { + return "There are no Rya instances installed on this storage."; + } + + // Sort the names alphabetically. Collections.sort( instanceNames ); + // Return a pretty print of the instances that have been installed. final String report; final InstanceNamesFormatter formatter = new InstanceNamesFormatter(); if(ryaInstance.isPresent()) {