Merge pull request #21 from mmedenjak/aws-discovery

Add discovery SPI implementation for AWS

src/main/java/com/hazelcast/aws/AwsDiscoveryStrategy.java

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.aws;
+
+import com.hazelcast.config.AwsConfig;
+import com.hazelcast.config.InvalidConfigurationException;
+import com.hazelcast.config.NetworkConfig;
+import com.hazelcast.logging.ILogger;
+import com.hazelcast.logging.Logger;
+import com.hazelcast.nio.Address;
+import com.hazelcast.spi.discovery.AbstractDiscoveryStrategy;
+import com.hazelcast.spi.discovery.DiscoveryNode;
+import com.hazelcast.spi.discovery.DiscoveryStrategy;
+import com.hazelcast.spi.discovery.SimpleDiscoveryNode;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Map;
+
+import static com.hazelcast.aws.AwsProperties.ACCESS_KEY;
+import static com.hazelcast.aws.AwsProperties.CONNECTION_TIMEOUT_SECONDS;
+import static com.hazelcast.aws.AwsProperties.HOST_HEADER;
+import static com.hazelcast.aws.AwsProperties.IAM_ROLE;
+import static com.hazelcast.aws.AwsProperties.PORT;
+import static com.hazelcast.aws.AwsProperties.REGION;
+import static com.hazelcast.aws.AwsProperties.SECRET_KEY;
+import static com.hazelcast.aws.AwsProperties.SECURITY_GROUP_NAME;
+import static com.hazelcast.aws.AwsProperties.TAG_KEY;
+import static com.hazelcast.aws.AwsProperties.TAG_VALUE;
+import static com.hazelcast.util.ExceptionUtil.rethrow;
+
+/**
+ * AWS implementation of {@link DiscoveryStrategy}.
+ *
+ * @see AWSClient
+ */
+public class AwsDiscoveryStrategy extends AbstractDiscoveryStrategy {
+    private static final ILogger LOGGER = Logger.getLogger(AwsDiscoveryStrategy.class);
+    private final AWSClient aws;
+    private final int port;
+
+    public AwsDiscoveryStrategy(Map<String, Comparable> properties) {
+        super(LOGGER, properties);
+        this.port = getOrDefault(PORT.getDefinition(), NetworkConfig.DEFAULT_PORT);
+        try {
+            this.aws = new AWSClient(getAwsConfig());
+        } catch (IllegalArgumentException e) {
+            throw new InvalidConfigurationException("AWS configuration is not valid", e);
+        }
+    }
+
+    private AwsConfig getAwsConfig() throws IllegalArgumentException {
+        final AwsConfig config = new AwsConfig()
+                .setEnabled(true)
+                .setAccessKey(getOrNull(ACCESS_KEY))
+                .setSecretKey(getOrNull(SECRET_KEY))
+                .setSecurityGroupName(getOrNull(SECURITY_GROUP_NAME))
+                .setTagKey(getOrNull(TAG_KEY))
+                .setTagValue(getOrNull(TAG_VALUE))
+                .setIamRole(getOrNull(IAM_ROLE));
+
+        final Integer timeout = getOrNull(CONNECTION_TIMEOUT_SECONDS.getDefinition());
+        if (timeout != null) {
+            config.setConnectionTimeoutSeconds(timeout);
+        }
+
+        final String region = getOrNull(REGION);
+        if (region != null) {
+            config.setRegion(region);
+        }
+
+        final String hostHeader = getOrNull(HOST_HEADER);
+        if (hostHeader != null) {
+            config.setHostHeader(hostHeader);
+        }
+        return config;
+    }
+
+    @Override
+    public Iterable<DiscoveryNode> discoverNodes() {
+        try {
+            final Map<String, String> privatePublicIpAddressPairs = aws.getAddresses();
+            if (privatePublicIpAddressPairs.isEmpty()) {
+                getLogger().warning("No EC2 instances found!");
+                return Collections.emptyList();
+            }
+
+            if (getLogger().isFinestEnabled()) {
+                final StringBuilder sb = new StringBuilder("Found the following EC2 instances:\n");
+                for (Map.Entry<String, String> entry : privatePublicIpAddressPairs.entrySet()) {
+                    sb.append("    ").append(entry.getKey()).append(" : ").append(entry.getValue()).append("\n");
+                }
+                getLogger().finest(sb.toString());
+            }
+
+            final ArrayList<DiscoveryNode> nodes = new ArrayList<DiscoveryNode>(privatePublicIpAddressPairs.size());
+            for (Map.Entry<String, String> entry : privatePublicIpAddressPairs.entrySet()) {
+                nodes.add(new SimpleDiscoveryNode(new Address(entry.getKey(), port), new Address(entry.getValue(), port)));
+            }
+
+            return nodes;
+        } catch (Exception e) {
+            throw rethrow(e);
+        }
+    }
+
+    private String getOrNull(AwsProperties awsProperties) {
+        return getOrNull(awsProperties.getDefinition());
+    }
+}

src/main/java/com/hazelcast/aws/AwsDiscoveryStrategyFactory.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.aws;
+
+import com.hazelcast.config.properties.PropertyDefinition;
+import com.hazelcast.logging.ILogger;
+import com.hazelcast.spi.discovery.DiscoveryNode;
+import com.hazelcast.spi.discovery.DiscoveryStrategy;
+import com.hazelcast.spi.discovery.DiscoveryStrategyFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Factory class which returns {@link AwsDiscoveryStrategy} to Discovery SPI
+ */
+public class AwsDiscoveryStrategyFactory implements DiscoveryStrategyFactory {
+    @Override
+    public Class<? extends DiscoveryStrategy> getDiscoveryStrategyType() {
+        return AwsDiscoveryStrategy.class;
+    }
+
+    @Override
+    public DiscoveryStrategy newDiscoveryStrategy(DiscoveryNode discoveryNode,
+                                                  ILogger logger,
+                                                  Map<String, Comparable> properties) {
+        return new AwsDiscoveryStrategy(properties);
+    }
+
+    @Override
+    public Collection<PropertyDefinition> getConfigurationProperties() {
+        final AwsProperties[] props = AwsProperties.values();
+        final ArrayList<PropertyDefinition> definitions = new ArrayList<PropertyDefinition>(props.length);
+        for (AwsProperties prop : props) {
+            definitions.add(prop.getDefinition());
+        }
+        return definitions;
+    }
+}

src/main/java/com/hazelcast/aws/AwsProperties.java

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2008-2016, Hazelcast, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.hazelcast.aws;
+
+import com.hazelcast.config.AwsConfig;
+import com.hazelcast.config.NetworkConfig;
+import com.hazelcast.config.TcpIpConfig;
+import com.hazelcast.config.properties.PropertyDefinition;
+import com.hazelcast.config.properties.PropertyTypeConverter;
+import com.hazelcast.config.properties.SimplePropertyDefinition;
+import com.hazelcast.config.properties.ValidationException;
+import com.hazelcast.config.properties.ValueValidator;
+
+import static com.hazelcast.config.properties.PropertyTypeConverter.INTEGER;
+import static com.hazelcast.config.properties.PropertyTypeConverter.STRING;
+
+/**
+ * Configuration properties for the Hazelcast Discovery Plugin for AWS. For more information
+ * see {@link AwsConfig}
+ */
+public enum AwsProperties {
+    /**
+     * Access key of your account on EC2
+     */
+    ACCESS_KEY("access-key", STRING, false),
+    /**
+     * Secret key of your account on EC2
+     */
+    SECRET_KEY("secret-key", STRING, false),
+    /**
+     * The region where your members are running. Default value is us-east-1. You need to specify this if the region is other
+     * than the default one.
+     */
+    REGION("region", STRING, true),
+    /**
+     * IAM roles are used to make secure requests from your clients. You can provide the name
+     * of your IAM role that you created previously on your AWS console.
+     */
+    IAM_ROLE("iam-role", STRING, true),
+    /**
+     * The URL that is the entry point for a web service (the address where the EC2 API can be found).
+     * Default value is ec2.amazonaws.com.
+     */
+    HOST_HEADER("host-header", STRING, true),
+    /**
+     * Name of the security group you specified at the EC2 management console. It is used to narrow the Hazelcast members to
+     * be within this group. It is optional.
+     */
+    SECURITY_GROUP_NAME("security-group-name", STRING, true),
+    /**
+     * Tag key as specified in the EC2 console. It is used to narrow the members returned by the discovery mechanism.
+     */
+    TAG_KEY("tag-key", STRING, true),
+    /**
+     * Tag value as specified in the EC2 console. It is used to narrow the members returned by the discovery mechanism.
+     */
+    TAG_VALUE("tag-value", STRING, true),
+    /**
+     * Sets the connect timeout in seconds. See {@link TcpIpConfig#setConnectionTimeoutSeconds(int)} for more information.
+     * Its default value is 5.
+     */
+    CONNECTION_TIMEOUT_SECONDS("connection-timeout-seconds", INTEGER, true),
+    /**
+     * The discovery mechanism will discover only IP addresses. You can define the port on which Hazelcast is expected to be
+     * running here. This port number is not used by the discovery mechanism itself, it is only returned by the discovery
+     * mechanism. The default port is {@link NetworkConfig#DEFAULT_PORT}
+     */
+    PORT("hz-port", INTEGER, true, new PortValueValidator());
+
+    private static final int MIN_PORT = 0;
+    private static final int MAX_PORT = 65535;
+    private final PropertyDefinition propertyDefinition;
+
+    AwsProperties(String key, PropertyTypeConverter typeConverter, boolean optional, ValueValidator validator) {
+        this.propertyDefinition = new SimplePropertyDefinition(key, optional, typeConverter, validator);
+    }
+
+    AwsProperties(String key, PropertyTypeConverter typeConverter, boolean optional) {
+        this.propertyDefinition = new SimplePropertyDefinition(key, optional, typeConverter);
+    }
+
+    public PropertyDefinition getDefinition() {
+        return propertyDefinition;
+    }
+
+    /**
+     * Validator for valid network ports
+     */
+    public static class PortValueValidator implements ValueValidator<Integer> {
+
+        /**
+         * Returns a validation
+         *
+         * @param value the integer to validate
+         * @throws ValidationException if value does not fall in valid port number range
+         */
+        public void validate(Integer value) throws ValidationException {
+            if (value < MIN_PORT) {
+                throw new ValidationException("hz-port number must be greater 0");
+            }
+            if (value > MAX_PORT) {
+                throw new ValidationException("hz-port number must be less or equal to 65535");
+            }
+        }
+    }
+}

src/main/java/com/hazelcast/aws/impl/DescribeInstances.java

@@ -182,7 +182,8 @@ private void getKeysFromIamRole() {
         }
     }
 
-    /** This helper method is responsible for just parsing the content of the HTTP response and
+    /**
+     * This helper method is responsible for just parsing the content of the HTTP response and
      * storing the access keys and token it finds there.
      *
      * @param json The JSON representation of the IAM (Task) Role.
@@ -195,11 +196,11 @@ private void parseAndStoreRoleCreds(String json) {
     }
 
     /**
-     * @deprecated Since we moved JSON parsing from manual pattern matching to using
-     * `com.hazelcast.com.eclipsesource.json.JsonObject`, this method should be deprecated.
      * @param reader The reader that gives access to the JSON-formatted content that includes all the role information.
      * @return A map with all the parsed keys and values from the JSON content.
      * @throws IOException In case the input from reader cannot be correctly parsed.
+     * @deprecated Since we moved JSON parsing from manual pattern matching to using
+     * `com.hazelcast.com.eclipsesource.json.JsonObject`, this method should be deprecated.
      */
     @Deprecated
     public Map<String, String> parseIamRole(BufferedReader reader) throws IOException {
@@ -227,6 +228,14 @@ private String getFormattedTimestamp() {
         return df.format(new Date());
     }
 
+    /**
+     * Invoke the service to describe the instances, unmarshal the response and return the discovered node map.
+     * The map contains mappings from private to public IP and all contained nodes match the filtering rules defined by
+     * the {@link #awsConfig}.
+     *
+     * @return map from private to public IP or empty map in case of failed response unmarshalling
+     * @throws Exception if there is an exception invoking the service
+     */
     public Map<String, String> execute() throws Exception {
         String signature = rs.sign("ec2", attributes);
         Map<String, String> response;

src/main/java/com/hazelcast/aws/utility/CloudyUtility.java

@@ -25,7 +25,6 @@
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
-import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
 import java.util.LinkedHashMap;
@@ -47,7 +46,17 @@ public final class CloudyUtility {
     private CloudyUtility() {
     }
 
-    public static Map<String, String> unmarshalTheResponse(InputStream stream, AwsConfig awsConfig) throws IOException {
+    /**
+     * Unmarshal the response from {@link com.hazelcast.aws.impl.DescribeInstances} and return the discovered node map.
+     * The map contains mappings from private to public IP and all contained nodes match the filtering rules defined by
+     * the {@code awsConfig}.
+     * If there is an exception while unmarshaling the response, returns an empty map.
+     *
+     * @param stream    the response XML stream
+     * @param awsConfig the AWS configuration for filtering the returned addresses
+     * @return map from private to public IP or empty map in case of exceptions
+     */
+    public static Map<String, String> unmarshalTheResponse(InputStream stream, AwsConfig awsConfig) {
         DocumentBuilder builder;
         try {
             DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
@@ -110,6 +119,15 @@ List<NodeHolder> getSubNodes(String name) {
             return result;
         }
 
+        /**
+         * Unmarshal the response from the {@link com.hazelcast.aws.impl.DescribeInstances} service and
+         * return the map from private to public IP. All returned entries must match filters defined by the {@code config}.
+         * This method expects that the DOM containing the XML has been positioned at the node containing the addresses.
+         *
+         * @param awsConfig the AWS configuration for filtering the returned addresses
+         * @return map from private to public IP
+         * @see #getFirstSubNode(String)
+         */
         Map<String, String> getAddresses(AwsConfig awsConfig) {
             Map<String, String> privatePublicPairs = new LinkedHashMap<String, String>();
             if (node == null) {

src/main/resources/META-INF/services/com.hazelcast.spi.discovery.DiscoveryStrategyFactory

@@ -0,0 +1 @@
+com.hazelcast.aws.AwsDiscoveryStrategyFactory