Permalink
Browse files

WHIRR-347. Support provider-independent environment variables for clo…

…ud credentials (asavu)

git-svn-id: https://svn.apache.org/repos/asf/whirr/trunk@1243842 13f79535-47bb-0310-9956-ffa450edef68
  • Loading branch information...
1 parent e5c8960 commit dcad614d1f1b07d5442d0b0e6e5a7b7d43e9066e @andreisavu andreisavu committed Feb 14, 2012
Showing with 349 additions and 318 deletions.
  1. +1 −0 .gitignore
  2. +3 −0 CHANGES.txt
  3. +20 −0 bin/whirr
  4. +1 −1 cli/src/main/java/org/apache/whirr/cli/command/CleanupClusterCommand.java
  5. +1 −1 cli/src/main/java/org/apache/whirr/cli/command/DestroyClusterCommand.java
  6. +53 −0 conf/credentials.sample
  7. +2 −1 core/src/main/java/org/apache/whirr/ClusterSpec.java
  8. +18 −14 core/src/main/java/org/apache/whirr/command/AbstractClusterCommand.java
  9. +4 −0 core/src/main/resources/whirr-default.properties
  10. +8 −8 core/src/test/java/org/apache/whirr/command/AbstractClusterCommandTest.java
  11. +7 −6 recipes/{cassandra-ec2.properties → cassandra.properties}
  12. +0 −91 recipes/elasticsearch-rackspace.properties
  13. +7 −6 recipes/{elasticsearch-ec2.properties → elasticsearch.properties}
  14. +3 −5 recipes/{ganglia-ec2.properties → ganglia.properties}
  15. +0 −52 recipes/hadoop-rackspace.properties
  16. +44 −15 recipes/{hadoop-ec2.properties → hadoop.properties}
  17. +35 −11 recipes/{hama-ec2.properties → hama.properties}
  18. +30 −12 recipes/{hbase-ec2-0.90.properties → hbase-0.90.properties}
  19. +25 −9 recipes/{hbase-ec2-cdh.properties → hbase-cdh.properties}
  20. +30 −12 recipes/{hbase-ec2.properties → hbase.properties}
  21. +34 −4 recipes/{mahout-ec2.properties → mahout.properties}
  22. +0 −50 recipes/puppet-http-rackspace.properties
  23. +2 −4 recipes/{puppet-http-ec2.properties → puppet-http.properties}
  24. +5 −7 recipes/{voldemort-ec2.properties → voldemort.properties}
  25. +3 −5 recipes/{zookeeper-ec2.properties → zookeeper.properties}
  26. +9 −2 src/site/xdoc/quick-start-guide.xml
  27. +4 −2 src/site/xdoc/whirr-in-5-minutes.xml
View
@@ -8,3 +8,4 @@ target
**/*.log
services/voldemort/lib/
whirr.log
+conf/credentials
View
@@ -53,6 +53,9 @@ Trunk (unreleased changes)
WHIRR-422. Integration tests should fail or succeed in a limited
amount of time (asavu)
+ WHIRR-347. Support provider-independent environment variables for
+ cloud credentials (asavu)
+
BUG FIXES
WHIRR-367. Wrong groupId for zookeeper (Joe Crobak via asavu)
View
@@ -24,6 +24,23 @@ case "`uname`" in
CYGWIN*) cygwin=true;
esac
+if [ -e ~/.whirr/credentials ]; then
+ . ~/.whirr/credentials
+elif [ -e $BIN/../conf/credentials ]; then
+ . $BIN/../conf/credentials
+fi
+
+# Override cloud provider credentials as needed
+export WHIRR_PROVIDER=${WHIRR_PROVIDER:-$PROVIDER}
+export WHIRR_IDENTITY=${WHIRR_IDENTITY:-$IDENTITY}
+export WHIRR_CREDENTIAL=${WHIRR_CREDENTIAL:-$CREDENTIAL}
+
+# Override blob store credentials as needed
+export WHIRR_BLOBSTORE_PROVIDER=${WHIRR_BLOBSTORE_PROVIDER:-$BLOBSTORE_PROVIDER}
+export WHIRR_BLOBSTORE_IDENTITY=${WHIRR_BLOBSTORE_IDENTITY:-$BLOBSTORE_IDENTITY}
+export WHIRR_BLOBSTORE_CREDENTIAL=${WHIRR_BLOBSTORE_CREDENTIAL:-$BLOBSTORE_CREDENTIAL}
+
+# Build the classpath
if [ -d "$BIN/../cli/target/lib" ]; then
CLASSPATH="$BIN/../conf/:$BIN/..:$BIN/../cli/target/lib/*:$BIN/../cli/target/*"
else
@@ -36,5 +53,8 @@ if $cygwin; then
CLASSPATH=`cygpath -wp $CLASSPATH`;
fi
+echo Starting cluster on "$WHIRR_PROVIDER" using identity "$WHIRR_IDENTITY"
+
+# Start the application
java $LOGGING_OPTS $WHIRR_CLI_OPTS -cp "$CLASSPATH" org.apache.whirr.cli.Main "$@"
@@ -52,7 +52,7 @@ public CleanupClusterCommand(ClusterControllerFactory factory,
public int run(InputStream in, PrintStream out, PrintStream err,
List<String> args) throws Exception {
- OptionSet optionSet = parser.parse(args.toArray(new String[0]));
+ OptionSet optionSet = parser.parse(args.toArray(new String[args.size()]));
if (!optionSet.nonOptionArguments().isEmpty()) {
printUsage(err);
return -1;
@@ -46,7 +46,7 @@ public DestroyClusterCommand(ClusterControllerFactory factory) {
public int run(InputStream in, PrintStream out, PrintStream err,
List<String> args) throws Exception {
- OptionSet optionSet = parser.parse(args.toArray(new String[0]));
+ OptionSet optionSet = parser.parse(args.toArray(new String[args.size()]));
if (!optionSet.nonOptionArguments().isEmpty()) {
printUsage(err);
View
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# In this file users can store their cloud login credentials for convenience
+# If this file exists it is sourced by Whirr scripts.
+#
+# Whirr will look for this file in the home directory first (~/.whirr/credentials)
+# If not found it will look for it in the conf directory.
+#
+# Preset environment variables will take precedence over these files.
+#
+# VARIABLES:
+#
+# PROVIDER - The cloud provider to use in Whirr
+# IDENTITY - The identity to use in Whirr
+# CREDENTIAL - The credential to use in Whirr
+#
+# BLOBSTORE_* - Overrides the base variables (e.g. PROVIDER) specifically for
+# blobstore contexts. Base variables are still used for compute access
+# If BLOBSTORE_* variables are not defined Whirr will use the base variables
+# for blobstore access.
+
+# Users can assign these variables the values they want to assign to
+# their WHIRR_* variable counterparts. If a WHIRR_* variable is found in env
+# then it takes precedence (to be able to do one-off overrides on recipes)
+# otherwise WHIRR_* variables take the value from this file. Finally .properties
+# files override both this file and previous env variables
+
+# Set cloud provider connection details
+
+PROVIDER=
+IDENTITY=
+CREDENTIAL=
+
+# Set blob store connection details. If not defined they are computed
+# from the cloud provider connection details defined above
+
+# BLOBSTORE_PROVIDER=
+# BLOBSTORE_IDENTITY=
+# BLOBSTORE_CREDENTIAL=
+
@@ -434,7 +434,8 @@ private Configuration composeWithDefaults(Configuration userConfig)
throws ConfigurationException {
CompositeConfiguration composed = new CompositeConfiguration();
composed.addConfiguration(userConfig);
- composed.addConfiguration(new PropertiesConfiguration(getClass().getClassLoader().getResource(DEFAULT_PROPERTIES)));
+ composed.addConfiguration(new PropertiesConfiguration(
+ getClass().getClassLoader().getResource(DEFAULT_PROPERTIES)));
return composed;
}
@@ -18,22 +18,15 @@
package org.apache.whirr.command;
-import static org.apache.whirr.ClusterSpec.Property.CLUSTER_NAME;
-import static org.apache.whirr.ClusterSpec.Property.IDENTITY;
-import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES;
-
import com.google.common.collect.Maps;
-
import java.io.IOException;
import java.io.PrintStream;
import java.util.EnumSet;
import java.util.Map;
-
import joptsimple.ArgumentAcceptingOptionSpec;
import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
-
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
@@ -43,6 +36,12 @@
import org.apache.whirr.ClusterControllerFactory;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.ClusterSpec.Property;
+import static org.apache.whirr.ClusterSpec.Property.CLUSTER_NAME;
+import static org.apache.whirr.ClusterSpec.Property.CREDENTIAL;
+import static org.apache.whirr.ClusterSpec.Property.IDENTITY;
+import static org.apache.whirr.ClusterSpec.Property.INSTANCE_TEMPLATES;
+import static org.apache.whirr.ClusterSpec.Property.PRIVATE_KEY_FILE;
+import static org.apache.whirr.ClusterSpec.Property.PROVIDER;
import org.apache.whirr.state.ClusterStateStore;
import org.apache.whirr.state.ClusterStateStoreFactory;
import org.slf4j.Logger;
@@ -100,12 +99,14 @@ protected ClusterSpec getClusterSpec(OptionSet optionSet) throws ConfigurationEx
for (Map.Entry<Property, OptionSpec<?>> entry : optionSpecs.entrySet()) {
Property property = entry.getKey();
OptionSpec<?> option = entry.getValue();
+ Object value;
if (property.hasMultipleArguments()) {
- optionsConfig.setProperty(property.getConfigName(),
- optionSet.valuesOf(option));
+ value = optionSet.valuesOf(option);
} else {
- optionsConfig.setProperty(property.getConfigName(),
- optionSet.valueOf(option));
+ value = optionSet.valueOf(option);
+ }
+ if (value != null) {
+ optionsConfig.setProperty(property.getConfigName(), value);
}
}
CompositeConfiguration config = new CompositeConfiguration();
@@ -114,14 +115,17 @@ protected ClusterSpec getClusterSpec(OptionSet optionSet) throws ConfigurationEx
Configuration defaults = new PropertiesConfiguration(optionSet.valueOf(configOption));
config.addConfiguration(defaults);
}
+ ClusterSpec clusterSpec = new ClusterSpec(config);
- for (Property required : EnumSet.of(CLUSTER_NAME, IDENTITY, INSTANCE_TEMPLATES)) {
- if (config.getString(required.getConfigName()) == null) {
+ for (Property required : EnumSet.of(CLUSTER_NAME, PROVIDER, IDENTITY, CREDENTIAL,
+ INSTANCE_TEMPLATES, PRIVATE_KEY_FILE)) {
+ if (clusterSpec.getConfiguration().getString(required.getConfigName()) == null) {
throw new IllegalArgumentException(String.format("Option '%s' not set.",
required.getSimpleName()));
}
}
- return new ClusterSpec(config);
+
+ return clusterSpec;
}
/**
@@ -15,6 +15,10 @@ whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
# whirr.public-key-file = ${whirr.private-key-file}.pub
# this is created in ClusterSpec class constructor
+whirr.provider=${env:WHIRR_PROVIDER}
+whirr.identity=${env:WHIRR_IDENTITY}
+whirr.credential=${env:WHIRR_CREDENTIAL}
+
whirr.version=${version}
whirr.max-startup-retries=1
@@ -18,21 +18,20 @@
package org.apache.whirr.command;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
+import joptsimple.OptionSet;
+import org.apache.whirr.ClusterControllerFactory;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.util.KeyPair;
+import org.junit.Test;
import java.io.File;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
import java.util.Map;
-import joptsimple.OptionSet;
-
-import org.apache.whirr.ClusterControllerFactory;
-import org.apache.whirr.ClusterSpec;
-import org.apache.whirr.util.KeyPair;
-import org.junit.Test;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
public class AbstractClusterCommandTest {
@@ -62,6 +61,7 @@ public int run(InputStream in, PrintStream out, PrintStream err,
* Ensure that an invalid service name uses the default (after logging a
* warning).
*/
+ @Test
public void testCreateServerWithInvalidClusterControllerName() throws Exception {
AbstractClusterCommand clusterCommand = new AbstractClusterCommand("name",
"description", new ClusterControllerFactory()) {
@@ -16,7 +16,7 @@
#
#
-# Cassandra Cluster on AWS EC2
+# Setup an Apache Cassandra Cluster
#
# Read the Configuration Guide for more info:
@@ -25,14 +25,15 @@
# Change the cluster name here
whirr.cluster-name=cassandra
+# Setup your cloud credentials by copying conf/credentials.sample
+# to ~/.whirr/credentials and editing as needed
+
+# Change the name of cluster admin user
+whirr.cluster-user=${sys:user.name}
+
# Change the number of machines in the cluster here
whirr.instance-templates=3 cassandra
-# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
-whirr.provider=aws-ec2
-whirr.identity=${env:AWS_ACCESS_KEY_ID}
-whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
-
# By default use the user system SSH keys. Override them here.
# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
# whirr.public-key-file=${whirr.private-key-file}.pub
@@ -1,91 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# ElasticSearch Cluster on Rackspace Cloud
-#
-
-# Read the Configuration Guide for more info:
-# http://whirr.apache.org/docs/latest/configuration-guide.html
-
-# Change the cluster name here
-whirr.cluster-name=elasticsearch
-
-# Change the number of machines in the cluster here
-whirr.instance-templates=2 elasticsearch
-
-# For Rackspace set RACKSPACE_USERNAME and RACKSPACE_API_KEY environment variables.
-whirr.provider=cloudservers-us
-whirr.identity=${env:RACKSPACE_USERNAME}
-whirr.credential=${env:RACKSPACE_API_KEY}
-
-# The size of the instance to use. See http://www.rackspacecloud.com/cloud_hosting_products/servers/faq/
-# id 3: 1GB, 1 virtual core
-# id 4: 2GB, 2 virtual cores
-# id 5: 4GB, 2 virtual cores
-# id 6: 8GB, 4 virtual cores
-# id 7: 15.5GB, 4 virtual cores
-whirr.hardware-id=6
-# Ubuntu 10.04 LTS Lucid
-whirr.image-id=49
-
-# By default use the user system SSH keys. Override them here.
-# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
-# whirr.public-key-file=${whirr.private-key-file}.pub
-
-# You can specify the version by setting the tarball url
-# whirr.elasticsearch.tarball.url=http://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.15.2.tar.gz
-
-#
-# elasticsearch specific settings (Expert)
-#
-
-# 1. Gateway Persistence settings
-# See: http://www.elasticsearch.org/guide/reference/modules/gateway/
-# Defaults: the index is only stored in memory and all data is lost on shutdown
-
-# 1.1 Enable persistence on the local filesystem
-# See: http://www.elasticsearch.org/guide/reference/modules/gateway/local.html
-
-# es.gateway.type=fs
-# es.gateway.recovery_after_nodes=1
-# es.gateway.recovery_after_time=5m
-# es.expected_nodes=2
-
-# 1.2 Enable persistence on HDFS
-# See: http://www.elasticsearch.org/guide/reference/modules/gateway/hadoop.html
-
-# es.gateway.type=hdfs
-# es.gateway.hdfs.uri=hdfs://myhost:8022
-# es.gateway.hdfs.path=/some/path
-
-# 2. Scripting Support
-# See: http://www.elasticsearch.org/guide/reference/modules/scripting.html
-# The scripting module uses by default mvel
-
-# Just add them to the list of installed plugins
-# es.plugins=lang-javascript, lang-groovy, lang-python,
-
-# 3. Memcached protocol support
-# See: http://www.elasticsearch.org/guide/reference/modules/memcached.html
-
-# es.plugins=transport-memcached
-
-# 4. Thrift protocol support
-# See: http://www.elasticsearch.org/guide/reference/modules/thrift.html
-
-# es.plugins=transport-thrift
Oops, something went wrong.

0 comments on commit dcad614

Please sign in to comment.