Ignite 2.3.2.b1

examples/pom-standalone-lgpl.xml

@@ -81,13 +81,7 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>2.8.4</version>
         </dependency>
     </dependencies>
 

examples/pom-standalone.xml

@@ -81,13 +81,7 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>2.8.4</version>
         </dependency>
     </dependencies>
 

examples/pom.xml

@@ -70,13 +70,7 @@
         <dependency>
             <groupId>com.google.code.simple-spring-memcached</groupId>
             <artifactId>spymemcached</artifactId>
-            <version>2.7.3</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>commons-codec</groupId>
-                    <artifactId>commons-codec</artifactId>
-                </exclusion>
-            </exclusions>
+            <version>2.8.4</version>
         </dependency>
 
         <dependency>
@@ -140,12 +134,6 @@
                     <artifactId>ignite-spark</artifactId>
                     <version>${project.version}</version>
                 </dependency>
-
-                <dependency>
-                    <groupId>org.jboss.netty</groupId>
-                    <artifactId>netty</artifactId>
-                    <version>3.2.9.Final</version>
-                </dependency>
             </dependencies>
 
             <build>

modules/aws/src/main/java/org/apache/ignite/spi/checkpoint/s3/S3CheckpointSpi.java

@@ -67,13 +67,15 @@
  * <h2 class="header">Mandatory</h2>
  * This SPI has one mandatory configuration parameter:
  * <ul>
- * <li>{@link #setAwsCredentials(AWSCredentials)}</li>
+ *      <li>AWS credentials (see {@link #setAwsCredentials(AWSCredentials)}
  * </ul>
  * <h2 class="header">Optional</h2>
  * This SPI has following optional configuration parameters:
  * <ul>
- * <li>{@link #setBucketNameSuffix(String)}</li>
- * <li>{@link #setClientConfiguration(ClientConfiguration)}</li>
+ *      <li>Bucket name suffix (see {@link #setBucketNameSuffix(String)})</li>
+ *      <li>Client configuration (see {@link #setClientConfiguration(ClientConfiguration)})</li>
+ *      <li>Bucket endpoint (see {@link #setBucketEndpoint(String)})</li>
+ *      <li>Server side encryption algorithm (see {@link #setSSEAlgorithm(String)})</li>
  * </ul>
  * <h2 class="header">Java Example</h2>
  * {@link S3CheckpointSpi} can be configured as follows:
@@ -155,6 +157,12 @@ public class S3CheckpointSpi extends IgniteSpiAdapter implements CheckpointSpi {
     /** Bucket name (generated). */
     private String bucketName;
 
+    /** Bucket endpoint (set by user). */
+    private @Nullable String bucketEndpoint;
+
+    /** Server side encryption algorithm */
+    private @Nullable String sseAlg;
+
     /** Amazon client configuration. */
     private ClientConfiguration cfg;
 
@@ -174,6 +182,24 @@ public String getBucketName() {
         return bucketName;
     }
 
+    /**
+     * Gets S3 bucket endpoint to use.
+     *
+     * @return S3 bucket endpoint to use.
+     */
+    public @Nullable String getBucketEndpoint() {
+        return bucketEndpoint;
+    }
+
+    /**
+     * Gets S3 server-side encryption algorithm.
+     *
+     * @return S3 server-side encryption algorithm to use.
+     */
+    public @Nullable String getSSEAlgorithm() {
+        return sseAlg;
+    }
+
     /**
      * Gets S3 access key.
      *
@@ -241,6 +267,37 @@ public S3CheckpointSpi setBucketNameSuffix(String bucketNameSuffix) {
         return this;
     }
 
+    /**
+     * Sets bucket endpoint.
+     * If the endpoint is not set then S3CheckpointSpi will go to each region to find a corresponding bucket.
+     * For information about possible endpoint names visit
+     * <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">docs.aws.amazon.com</a>
+     *
+     * @param bucketEndpoint Bucket endpoint, for example, {@code }s3.us-east-2.amazonaws.com.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public S3CheckpointSpi setBucketEndpoint(String bucketEndpoint) {
+        this.bucketEndpoint = bucketEndpoint;
+
+        return this;
+    }
+
+    /**
+     * Sets server-side encryption algorithm for Amazon S3-managed encryption keys.
+     * For information about possible S3-managed encryption keys visit
+     * <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html">docs.aws.amazon.com</a>.
+     *
+     * @param sseAlg Server-side encryption algorithm, for example, AES256 or SSES3.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public S3CheckpointSpi setSSEAlgorithm(String sseAlg) {
+        this.sseAlg = sseAlg;
+
+        return this;
+    }
+
     /**
      * Sets Amazon client configuration.
      * <p>
@@ -283,6 +340,8 @@ public S3CheckpointSpi setAwsCredentials(AWSCredentials cred) {
             log.debug(configInfo("awsCredentials", cred));
             log.debug(configInfo("clientConfiguration", cfg));
             log.debug(configInfo("bucketNameSuffix", bucketNameSuffix));
+            log.debug(configInfo("bucketEndpoint", bucketEndpoint));
+            log.debug(configInfo("SSEAlgorithm", sseAlg));
         }
 
         if (cfg == null)
@@ -298,6 +357,9 @@ public S3CheckpointSpi setAwsCredentials(AWSCredentials cred) {
 
         s3 = cfg != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(cred);
 
+        if (!F.isEmpty(bucketEndpoint))
+            s3.setEndpoint(bucketEndpoint);
+
         if (!s3.doesBucketExist(bucketName)) {
             try {
                 s3.createBucket(bucketName);
@@ -528,6 +590,9 @@ private void write(S3CheckpointData data) throws IgniteCheckedException, AmazonC
 
         meta.setContentLength(buf.length);
 
+        if (!F.isEmpty(sseAlg))
+            meta.setSSEAlgorithm(sseAlg);
+
         s3.putObject(bucketName, data.getKey(), new ByteArrayInputStream(buf), meta);
     }
 
@@ -734,6 +799,16 @@ private class S3CheckpointSpiMBeanImpl extends IgniteSpiMBeanAdapter implements
             return S3CheckpointSpi.this.getBucketName();
         }
 
+        /** {@inheritDoc} */
+        @Override public String getBucketEndpoint() {
+            return S3CheckpointSpi.this.getBucketName();
+        }
+
+        /** {@inheritDoc} */
+        @Override public String getSSEAlgorithm() {
+            return S3CheckpointSpi.this.getSSEAlgorithm();
+        }
+
         /** {@inheritDoc} */
         @Override public String getAccessKey() {
             return S3CheckpointSpi.this.getAccessKey();

modules/aws/src/main/java/org/apache/ignite/spi/checkpoint/s3/S3CheckpointSpiMBean.java

@@ -33,6 +33,18 @@ public interface S3CheckpointSpiMBean extends IgniteSpiManagementMBean {
     @MXBeanDescription("S3 bucket name.")
     public String getBucketName();
 
+    /**
+     * @return S3 bucket endpoint.
+     */
+    @MXBeanDescription("S3 bucket endpoint.")
+    public String getBucketEndpoint();
+
+    /**
+     * @return S3 server-side encryption algorithm.
+     */
+    @MXBeanDescription("S3 server-side encryption algorithm.")
+    public String getSSEAlgorithm();
+
     /**
      * @return S3 access key.
      */

modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java

@@ -44,6 +44,7 @@
 import org.apache.ignite.spi.IgniteSpiConfiguration;
 import org.apache.ignite.spi.IgniteSpiException;
 import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter;
+import org.jetbrains.annotations.Nullable;
 
 /**
  * AWS S3-based IP finder.
@@ -60,6 +61,8 @@
  * <ul>
  *      <li>Client configuration (see {@link #setClientConfiguration(ClientConfiguration)})</li>
  *      <li>Shared flag (see {@link #setShared(boolean)})</li>
+ *      <li>Bucket endpoint (see {@link #setBucketEndpoint(String)})</li>
+ *      <li>Server side encryption algorithm (see {@link #setSSEAlgorithm(String)})</li>
  * </ul>
  * <p>
  * The finder will create S3 bucket with configured name. The bucket will contain entries named
@@ -78,14 +81,9 @@ public class TcpDiscoveryS3IpFinder extends TcpDiscoveryIpFinderAdapter {
     /** Entry content. */
     private static final byte[] ENTRY_CONTENT = new byte[] {1};
 
-    /** Entry metadata with content length set. */
-    private static final ObjectMetadata ENTRY_METADATA;
-
-    static {
-        ENTRY_METADATA = new ObjectMetadata();
-
-        ENTRY_METADATA.setContentLength(ENTRY_CONTENT.length);
-    }
+    /** Entry metadata. */
+    @GridToStringExclude
+    private final ObjectMetadata objMetadata = new ObjectMetadata();
 
     /** Grid logger. */
     @LoggerResource
@@ -98,6 +96,12 @@ public class TcpDiscoveryS3IpFinder extends TcpDiscoveryIpFinderAdapter {
     /** Bucket name. */
     private String bucketName;
 
+    /** Bucket endpoint */
+    private @Nullable String bucketEndpoint;
+
+    /** Server side encryption algorithm */
+    private @Nullable String sseAlg;
+
     /** Init guard. */
     @GridToStringExclude
     private final AtomicBoolean initGuard = new AtomicBoolean();
@@ -187,7 +191,7 @@ public TcpDiscoveryS3IpFinder() {
             String key = key(addr);
 
             try {
-                s3.putObject(bucketName, key, new ByteArrayInputStream(ENTRY_CONTENT), ENTRY_METADATA);
+                s3.putObject(bucketName, key, new ByteArrayInputStream(ENTRY_CONTENT), objMetadata);
             }
             catch (AmazonClientException e) {
                 throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName +
@@ -251,6 +255,11 @@ private void initClient() throws IgniteSpiException {
                 if (F.isEmpty(bucketName))
                     throw new IgniteSpiException("Bucket name is null or empty (provide bucket name and restart).");
 
+                objMetadata.setContentLength(ENTRY_CONTENT.length);
+
+                if (!F.isEmpty(sseAlg))
+                    objMetadata.setSSEAlgorithm(sseAlg);
+
                 s3 = createAmazonS3Client();
 
                 if (!s3.doesBucketExist(bucketName)) {
@@ -299,9 +308,14 @@ private void initClient() throws IgniteSpiException {
      * @return Client instance to use to connect to AWS.
      */
     private AmazonS3Client createAmazonS3Client() {
-        return cfg != null
+        AmazonS3Client cln = cfg != null
             ? (cred != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(credProvider, cfg))
             : (cred != null ? new AmazonS3Client(cred) : new AmazonS3Client(credProvider));
+
+        if (!F.isEmpty(bucketEndpoint))
+            cln.setEndpoint(bucketEndpoint);
+
+        return cln;
     }
 
     /**
@@ -317,6 +331,37 @@ public TcpDiscoveryS3IpFinder setBucketName(String bucketName) {
         return this;
     }
 
+    /**
+     * Sets bucket endpoint for IP finder.
+     * If the endpoint is not set then IP finder will go to each region to find a corresponding bucket.
+     * For information about possible endpoint names visit
+     * <a href="http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region">docs.aws.amazon.com</a>.
+     *
+     * @param bucketEndpoint Bucket endpoint, for example, s3.us-east-2.amazonaws.com.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public TcpDiscoveryS3IpFinder setBucketEndpoint(String bucketEndpoint) {
+        this.bucketEndpoint = bucketEndpoint;
+
+        return this;
+    }
+
+    /**
+     * Sets server-side encryption algorithm for Amazon S3-managed encryption keys.
+     * For information about possible S3-managed encryption keys visit
+     * <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html">docs.aws.amazon.com</a>.
+     *
+     * @param sseAlg Server-side encryption algorithm, for example, AES256 or SSES3.
+     * @return {@code this} for chaining.
+     */
+    @IgniteSpiConfiguration(optional = true)
+    public TcpDiscoveryS3IpFinder setSSEAlgorithm(String sseAlg) {
+        this.sseAlg = sseAlg;
+
+        return this;
+    }
+
     /**
      * Sets Amazon client configuration.
      * <p>

modules/aws/src/test/java/org/apache/ignite/spi/checkpoint/s3/S3CheckpointManagerSelfTest.java

@@ -41,7 +41,7 @@ public class S3CheckpointManagerSelfTest extends GridCheckpointManagerAbstractSe
 
         spi.setAwsCredentials(cred);
 
-        spi.setBucketNameSuffix("unit-test-bucket");
+        spi.setBucketNameSuffix(S3CheckpointSpiSelfTest.getBucketNameSuffix());
 
         cfg.setCheckpointSpi(spi);
 

modules/aws/src/test/java/org/apache/ignite/spi/checkpoint/s3/S3CheckpointSpiSelfTest.java

@@ -24,6 +24,9 @@
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.concurrent.ThreadLocalRandom;
 import org.apache.ignite.GridTestIoUtils;
 import org.apache.ignite.IgniteCheckedException;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
@@ -55,7 +58,7 @@ public class S3CheckpointSpiSelfTest extends GridSpiAbstractTest<S3CheckpointSpi
 
         spi.setAwsCredentials(cred);
 
-        spi.setBucketNameSuffix("unit-test-bucket");
+        spi.setBucketNameSuffix(getBucketNameSuffix());
 
         super.spiConfigure(spi);
     }
@@ -222,4 +225,25 @@ public void testDuplicates() throws Exception {
     private void assertWithRetries(GridAbsClosureX assertion) throws IgniteInterruptedCheckedException {
         GridTestUtils.retryAssert(log, 6, 5000, assertion);
     }
+
+    /**
+     * Gets a Bucket name suffix
+     * Bucket name suffix should be unique for the host to parallel test run on one bucket.
+     * Please note that the final bucket name should not exceed 63 chars.
+     *
+     * @return Bucket name suffix.
+     */
+    static String getBucketNameSuffix() {
+        String bucketNameSuffix;
+        try {
+            bucketNameSuffix = IgniteS3TestSuite.getBucketName(
+                "unit-test-" + InetAddress.getLocalHost().getHostName().toLowerCase());
+        }
+        catch (UnknownHostException e) {
+            bucketNameSuffix = IgniteS3TestSuite.getBucketName(
+                "unit-test-rnd-" + ThreadLocalRandom.current().nextInt(100));
+        }
+
+        return bucketNameSuffix;
+    }
 }