From 597ef0ba939e56720eae2a223de650664b91e271 Mon Sep 17 00:00:00 2001 From: Rahul Date: Mon, 9 Apr 2018 15:17:21 -0700 Subject: [PATCH 01/27] AWS S3 IP Finder SPI , replace depricated AWS APIs with new API added support to include public IP (for public address resolution) Added new S3 Address resolution depends on update S3 IP Finders SPI Add Consul IP Finder SPI and address resolvers --- .../configuration/S3AddressResolver.java | 140 ++++ .../ipfinder/s3/TcpDiscoveryS3IpFinder.java | 651 ++++++++++++------ modules/consul/licenses/apache-2.0.txt | 202 ++++++ modules/consul/pom.xml | 117 ++++ .../configuration/ConsulAddressResolver.java | 86 +++ .../consul/TcpDiscoveryConsulIpFinder.java | 312 +++++++++ .../tcp/ipfinder/consul/package-info.java | 22 + .../ipfinder/consul/util/PublicIpService.java | 45 ++ ...scoveryConsulIpFinderAbstractSelfTest.java | 102 +++ ...iscoveryConsulIpFinderHostUrlSelfTest.java | 44 ++ .../tcp/ipfinder/consul/package-info.java | 22 + .../testsuites/IgniteConsulTestSuite.java | 56 ++ pom.xml | 1 + 13 files changed, 1578 insertions(+), 222 deletions(-) create mode 100644 modules/aws/src/main/java/org/apache/ignite/configuration/S3AddressResolver.java create mode 100644 modules/consul/licenses/apache-2.0.txt create mode 100644 modules/consul/pom.xml create mode 100644 modules/consul/src/main/java/org/apache/ignite/configuration/ConsulAddressResolver.java create mode 100644 modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinder.java create mode 100644 modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java create mode 100644 modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/util/PublicIpService.java create mode 100644 modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderAbstractSelfTest.java create mode 100644 modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderHostUrlSelfTest.java create mode 100644 modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java create mode 100644 modules/consul/src/test/java/org/apache/ignite/testsuites/IgniteConsulTestSuite.java diff --git a/modules/aws/src/main/java/org/apache/ignite/configuration/S3AddressResolver.java b/modules/aws/src/main/java/org/apache/ignite/configuration/S3AddressResolver.java new file mode 100644 index 0000000000000..a6de73ec549db --- /dev/null +++ b/modules/aws/src/main/java/org/apache/ignite/configuration/S3AddressResolver.java @@ -0,0 +1,140 @@ +package org.apache.ignite.configuration; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.util.IOUtils; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.SB; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.IgniteSpiException; +import org.jetbrains.annotations.Nullable; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.*; + +public class S3AddressResolver implements AddressResolver { + public static final String DELIM = "#"; + private Map inetSockAddrMap = new HashMap<>(); + private AWSCredentials cred; + private AWSCredentialsProvider credProvider; + private AmazonS3 s3; + private String bucketName; + private boolean doCache = true; + @Nullable + private String bucketEndpoint; + @Nullable +// private String sseAlg; +// @Nullable + private String region; + + @LoggerResource + private IgniteLogger log; + + public S3AddressResolver(AWSCredentials cred, String bucketName, @Nullable String bucketEndpoint) { + this(cred, bucketName, bucketEndpoint, true); + } + + public S3AddressResolver(AWSCredentialsProvider credProvider, String bucketName, @Nullable String bucketEndpoint) { + this(credProvider, bucketName, bucketEndpoint, true); + } + + public S3AddressResolver(AWSCredentials cred, String bucketName, @Nullable String bucketEndpoint, boolean doCache) { + this.cred = cred; + if (this.credProvider == null) { + this.credProvider = new AWSStaticCredentialsProvider(this.cred); + } + this.bucketName = bucketName; + this.bucketEndpoint = bucketEndpoint; + this.doCache = doCache; + //this.sseAlg = sseAlg; + init(); + } + + public S3AddressResolver(AWSCredentialsProvider credProvider, String bucketName, @Nullable String bucketEndpoint, boolean doCache) { + this.credProvider = credProvider; + this.bucketName = bucketName; + this.bucketEndpoint = bucketEndpoint; + this.doCache = doCache; + //this.sseAlg = sseAlg; + init(); + } + + private void init() { + if (this.cred == null && this.credProvider == null) { + throw new IgniteSpiException("AWS credentials are not set."); + } + if (F.isEmpty(this.bucketName)) { + throw new IgniteSpiException("Bucket name is null or empty (provide bucket name and restart)."); + } + + if (F.isEmpty(this.region)) { + this.region = Regions.US_WEST_1.getName(); + } + + this.s3 = this.createAmazonS3Client(); + if (!this.s3.doesBucketExist(this.bucketName)) + throw new IgniteSpiException("Bucket not exists [Bucket =" + bucketName); + + } + + + private String key(InetSocketAddress addr) { + assert addr != null; + SB sb = new SB(); + sb.a(addr.getAddress().getHostAddress()).a(DELIM).a(addr.getPort()); + return sb.toString(); + } + + private AmazonS3 createAmazonS3Client() { + + AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withRegion(region) + .withCredentials(credProvider) + .build(); + if (!F.isEmpty(this.bucketEndpoint)) { + s3Client.setEndpoint(this.bucketEndpoint); + } + return s3Client; + } + + private InetSocketAddress getResolvedAddress(InetSocketAddress inetSocketAddress) throws IgniteCheckedException { + InetSocketAddress mapAddress = null; + if (doCache) { + mapAddress = inetSockAddrMap.get(inetSocketAddress); + } + + if (!Optional.of(mapAddress).isPresent()) { + S3ObjectInputStream s3ObjectInputStream = null; + try { + s3ObjectInputStream = s3.getObject(this.bucketName, key(inetSocketAddress)).getObjectContent(); + mapAddress = InetSocketAddress.createUnresolved(new String(IOUtils.toByteArray(s3ObjectInputStream)), inetSocketAddress.getPort()); + if (doCache) + inetSockAddrMap.put(inetSocketAddress, mapAddress); + } catch (IOException e) { + throw new IgniteCheckedException(e); + } finally { + if (s3ObjectInputStream != null) { + try { + s3ObjectInputStream.close(); + } catch (IOException e) { + log.warning("S3 getObject stream closing failed."); + } + } + } + } + return mapAddress; + } + + @Override + public Collection getExternalAddresses(InetSocketAddress inetSocketAddress) throws IgniteCheckedException { + return Collections.singleton(getResolvedAddress(inetSocketAddress)); + } +} diff --git a/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java b/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java index 79559e8b6d86c..bdc950d49b65c 100644 --- a/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java +++ b/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java @@ -21,8 +21,8 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.regions.Region; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; @@ -45,6 +45,15 @@ import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter; import org.jetbrains.annotations.Nullable; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; +import java.io.IOException; +import java.util.*; /** * AWS S3-based IP finder. @@ -53,16 +62,17 @@ *

Configuration

*

Mandatory

*
    - *
  • AWS credentials (see {@link #setAwsCredentials(AWSCredentials)} and - * {@link #setAwsCredentialsProvider(AWSCredentialsProvider)}
    • - *
  • Bucket name (see {@link #setBucketName(String)})
    • + *
  • AWS credentials (see {@link #setAwsCredentials(AWSCredentials)} and + * {@link #setAwsCredentialsProvider(AWSCredentialsProvider)}
    • + *
  • Bucket name (see {@link #setBucketName(String)})
    • *
*

Optional

*
    - *
  • Client configuration (see {@link #setClientConfiguration(ClientConfiguration)})
    • - *
  • Shared flag (see {@link #setShared(boolean)})
    • - *
  • Bucket endpoint (see {@link #setBucketEndpoint(String)})
    • - *
  • Server side encryption algorithm (see {@link #setSSEAlgorithm(String)})
    • + *
  • Client configuration (see {@link #setClientConfiguration(ClientConfiguration)})
    • + *
  • Shared flag (see {@link #setShared(boolean)})
    • + *
  • Bucket endpoint (see {@link #setBucketEndpoint(String)})
    • + *
  • Region (see {@link #setRegion(String)})
    • + *
  • Server side encryption algorithm (see {@link #setSSEAlgorithm(String)})
    • *
*

* The finder will create S3 bucket with configured name. The bucket will contain entries named @@ -74,348 +84,545 @@ *

* Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}. */ -public class TcpDiscoveryS3IpFinder extends TcpDiscoveryIpFinderAdapter { - /** Delimiter to use in S3 entries name. */ - public static final String DELIM = "#"; - - /** Entry content. */ - private static final byte[] ENTRY_CONTENT = new byte[] {1}; - /** Entry metadata. */ +public class TcpDiscoveryS3IpFinder extends TcpDiscoveryIpFinderAdapter { + private static final String DELIM = "#"; + @GridToStringExclude + private static HttpClient httpClient = HttpClientBuilder.create().build(); @GridToStringExclude private final ObjectMetadata objMetadata = new ObjectMetadata(); - - /** Grid logger. */ + @GridToStringExclude + private final AtomicBoolean initGuard = new AtomicBoolean(); + @GridToStringExclude + private final CountDownLatch initLatch = new CountDownLatch(1); + private byte[] entryContent = new byte[]{1}; @LoggerResource private IgniteLogger log; - - /** Client to interact with S3 storage. */ @GridToStringExclude private AmazonS3 s3; - - /** Bucket name. */ private String bucketName; - - /** Bucket endpoint */ - private @Nullable String bucketEndpoint; - - /** Server side encryption algorithm */ - private @Nullable String sseAlg; - - /** Init guard. */ - @GridToStringExclude - private final AtomicBoolean initGuard = new AtomicBoolean(); - - /** Init latch. */ @GridToStringExclude - private final CountDownLatch initLatch = new CountDownLatch(1); - - /** Amazon client configuration. */ + private boolean isPublicIpMapRequired = false; + @Nullable + private String bucketEndpoint; + @Nullable + private String region; + @Nullable + private String sseAlg; private ClientConfiguration cfg; - - /** AWS Credentials. */ @GridToStringExclude private AWSCredentials cred; - - /** AWS Credentials. */ @GridToStringExclude private AWSCredentialsProvider credProvider; - /** - * Constructor. - */ public TcpDiscoveryS3IpFinder() { - setShared(true); + this.setShared(true); } - /** {@inheritDoc} */ - @Override public Collection getRegisteredAddresses() throws IgniteSpiException { - initClient(); - - Collection addrs = new LinkedList<>(); + public Collection getRegisteredAddresses() throws IgniteSpiException { + this.initClient(); + LinkedList addrs = new LinkedList(); try { - ObjectListing list = s3.listObjects(bucketName); + ObjectListing list = this.s3.listObjects(this.bucketName); while (true) { - for (S3ObjectSummary sum : list.getObjectSummaries()) { - String key = sum.getKey(); + Iterator var3 = list.getObjectSummaries().iterator(); + while (var3.hasNext()) { + S3ObjectSummary sum = (S3ObjectSummary) var3.next(); + String key = sum.getKey(); StringTokenizer st = new StringTokenizer(key, DELIM); - - if (st.countTokens() != 2) - U.error(log, "Failed to parse S3 entry due to invalid format: " + key); - else { + if (st.countTokens() != 2) { + U.error(this.log, "Failed to parse S3 entry due to invalid format: " + key); + } else { String addrStr = st.nextToken(); String portStr = st.nextToken(); - int port = -1; try { port = Integer.parseInt(portStr); - } - catch (NumberFormatException e) { - U.error(log, "Failed to parse port for S3 entry: " + key, e); + } catch (NumberFormatException var12) { + U.error(this.log, "Failed to parse port for S3 entry: " + key, var12); } - if (port != -1) + if (port != -1) { try { addrs.add(new InetSocketAddress(addrStr, port)); + } catch (IllegalArgumentException var11) { + U.error(this.log, "Failed to parse port for S3 entry: " + key, var11); } - catch (IllegalArgumentException e) { - U.error(log, "Failed to parse port for S3 entry: " + key, e); - } + } } } - if (list.isTruncated()) - list = s3.listNextBatchOfObjects(list); - else - break; + if (!list.isTruncated()) { + return addrs; + } + + list = this.s3.listNextBatchOfObjects(list); } + } catch (AmazonClientException var13) { + throw new IgniteSpiException("Failed to list objects in the bucket: " + this.bucketName, var13); } - catch (AmazonClientException e) { - throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); - } - - return addrs; } - /** {@inheritDoc} */ - @Override public void registerAddresses(Collection addrs) throws IgniteSpiException { + public void registerAddresses(Collection addrs) throws IgniteSpiException { assert !F.isEmpty(addrs); - initClient(); - - for (InetSocketAddress addr : addrs) { - String key = key(addr); + this.initClient(); + Iterator var2 = addrs.iterator(); + while (var2.hasNext()) { + InetSocketAddress addr = (InetSocketAddress) var2.next(); + String key = this.key(addr); try { - s3.putObject(bucketName, key, new ByteArrayInputStream(ENTRY_CONTENT), objMetadata); - } - catch (AmazonClientException e) { - throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName + - ", entry=" + key + ']', e); + this.s3.putObject(this.bucketName, key, new ByteArrayInputStream(getEntryContents()), this.objMetadata); + } catch (AmazonClientException var6) { + throw new IgniteSpiException("Failed to put entry [bucketName=" + this.bucketName + ", entry=" + key + ']', var6); } } + } - /** {@inheritDoc} */ - @Override public void unregisterAddresses(Collection addrs) throws IgniteSpiException { + public void unregisterAddresses(Collection addrs) throws IgniteSpiException { assert !F.isEmpty(addrs); - initClient(); + this.initClient(); + Iterator var2 = addrs.iterator(); - for (InetSocketAddress addr : addrs) { - String key = key(addr); + while (var2.hasNext()) { + InetSocketAddress addr = (InetSocketAddress) var2.next(); + String key = this.key(addr); try { - s3.deleteObject(bucketName, key); + this.s3.deleteObject(this.bucketName, key); + } catch (AmazonClientException var6) { + throw new IgniteSpiException("Failed to delete entry [bucketName=" + this.bucketName + ", entry=" + key + ']', var6); } - catch (AmazonClientException e) { - throw new IgniteSpiException("Failed to delete entry [bucketName=" + bucketName + - ", entry=" + key + ']', e); + } + + } + + private byte[] getEntryContents() { + try { + if (isPublicIpMapRequired) { + entryContent = httpClient.execute(new HttpGet("http://169.254.169.254/latest/meta-data/public-ipv4")).getEntity().getContent().toString().getBytes(); } + return entryContent; + } catch (ClientProtocolException e) { + throw new IgniteSpiException("Failed to get public IP", e); + } catch (IOException e) { + throw new IgniteSpiException("Failed to get public IP", e); } } - /** - * Gets S3 key for provided address. - * - * @param addr Node address. - * @return Key. - */ private String key(InetSocketAddress addr) { assert addr != null; SB sb = new SB(); - - sb.a(addr.getAddress().getHostAddress()) - .a(DELIM) - .a(addr.getPort()); - + sb.a(addr.getAddress().getHostAddress()).a(DELIM).a(addr.getPort()); return sb.toString(); } - /** - * Amazon s3 client initialization. - * - * @throws org.apache.ignite.spi.IgniteSpiException In case of error. - */ - @SuppressWarnings({"BusyWait"}) private void initClient() throws IgniteSpiException { - if (initGuard.compareAndSet(false, true)) + if (this.initGuard.compareAndSet(false, true)) { try { - if (cred == null && credProvider == null) + if (this.cred == null && this.credProvider == null) { throw new IgniteSpiException("AWS credentials are not set."); + } - if (cfg == null) - U.warn(log, "Amazon client configuration is not set (will use default)."); + if (this.cfg == null) { + U.warn(this.log, "Amazon client configuration is not set (will use default)."); + } - if (F.isEmpty(bucketName)) + if (F.isEmpty(this.bucketName)) { throw new IgniteSpiException("Bucket name is null or empty (provide bucket name and restart)."); + } - objMetadata.setContentLength(ENTRY_CONTENT.length); - - if (!F.isEmpty(sseAlg)) - objMetadata.setSSEAlgorithm(sseAlg); + this.isPublicIpMapRequired = Optional.of(this.isPublicIpMapRequired).orElse(false); - s3 = createAmazonS3Client(); + this.objMetadata.setContentLength((long) getEntryContents().length); + if (!F.isEmpty(this.sseAlg)) { + this.objMetadata.setSSEAlgorithm(this.sseAlg); + } - if (!s3.doesBucketExist(bucketName)) { + this.s3 = this.createAmazonS3Client(); + if (!this.s3.doesBucketExist(this.bucketName)) { try { - s3.createBucket(bucketName); - - if (log.isDebugEnabled()) - log.debug("Created S3 bucket: " + bucketName); + this.s3.createBucket(this.bucketName); + if (this.log.isDebugEnabled()) { + this.log.debug("Created S3 bucket: " + this.bucketName); + } - while (!s3.doesBucketExist(bucketName)) + while (!this.s3.doesBucketExist(this.bucketName)) { try { - U.sleep(200); - } - catch (IgniteInterruptedCheckedException e) { - throw new IgniteSpiException("Thread has been interrupted.", e); + U.sleep(200L); + } catch (IgniteInterruptedCheckedException var8) { + throw new IgniteSpiException("Thread has been interrupted.", var8); } - } - catch (AmazonClientException e) { - if (!s3.doesBucketExist(bucketName)) { - s3 = null; - - throw new IgniteSpiException("Failed to create bucket: " + bucketName, e); + } + } catch (AmazonClientException var9) { + if (!this.s3.doesBucketExist(this.bucketName)) { + this.s3 = null; + throw new IgniteSpiException("Failed to create bucket: " + this.bucketName, var9); } } } + } finally { + this.initLatch.countDown(); } - finally { - initLatch.countDown(); - } - else { + } else { try { - U.await(initLatch); - } - catch (IgniteInterruptedCheckedException e) { - throw new IgniteSpiException("Thread has been interrupted.", e); + U.await(this.initLatch); + } catch (IgniteInterruptedCheckedException var7) { + throw new IgniteSpiException("Thread has been interrupted.", var7); } - if (s3 == null) + if (this.s3 == null) { throw new IgniteSpiException("Ip finder has not been initialized properly."); + } } + } - /** - * Instantiates {@code AmazonS3Client} instance. - * - * @return Client instance to use to connect to AWS. - */ - private AmazonS3Client createAmazonS3Client() { - AmazonS3Client cln = cfg != null - ? (cred != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(credProvider, cfg)) - : (cred != null ? new AmazonS3Client(cred) : new AmazonS3Client(credProvider)); + private AmazonS3 createAmazonS3Client() { + if (this.credProvider == null) { + this.credProvider = new AWSStaticCredentialsProvider(this.cred); + } + AmazonS3 s3Client = AmazonS3ClientBuilder.standard() + .withRegion(region) + .withCredentials(credProvider) + .build(); + if (!F.isEmpty(this.bucketEndpoint)) { + s3Client.setEndpoint(this.bucketEndpoint); + } + + if (!F.isEmpty(this.region)) { + s3Client.setRegion(Region.getRegion(Regions.fromName(this.region))); + + + + + + + + + + + + + + + + + + + + - if (!F.isEmpty(bucketEndpoint)) - cln.setEndpoint(bucketEndpoint); - return cln; + + + + + + + + + + + + + + + + + + + + + + + + } + + + return s3Client; + } + + + @IgniteSpiConfiguration( + optional = true + ) + public TcpDiscoveryS3IpFinder setRegion(String region) { + this.region = region; + return this; } - /** - * Sets bucket name for IP finder. - * - * @param bucketName Bucket name. - * @return {@code this} for chaining. - */ - @IgniteSpiConfiguration(optional = false) + + @IgniteSpiConfiguration( + optional = true + ) + public TcpDiscoveryS3IpFinder setPublicIpMapRequired(boolean publicIpMapRequired) { + isPublicIpMapRequired = publicIpMapRequired; + return this; + } + + @IgniteSpiConfiguration( + optional = false + ) public TcpDiscoveryS3IpFinder setBucketName(String bucketName) { this.bucketName = bucketName; - return this; } - /** - * Sets bucket endpoint for IP finder. - * If the endpoint is not set then IP finder will go to each region to find a corresponding bucket. - * For information about possible endpoint names visit - * docs.aws.amazon.com. - * - * @param bucketEndpoint Bucket endpoint, for example, s3.us-east-2.amazonaws.com. - * @return {@code this} for chaining. - */ - @IgniteSpiConfiguration(optional = true) + @IgniteSpiConfiguration( + optional = true + ) public TcpDiscoveryS3IpFinder setBucketEndpoint(String bucketEndpoint) { this.bucketEndpoint = bucketEndpoint; - return this; } - /** - * Sets server-side encryption algorithm for Amazon S3-managed encryption keys. - * For information about possible S3-managed encryption keys visit - * docs.aws.amazon.com. - * - * @param sseAlg Server-side encryption algorithm, for example, AES256 or SSES3. - * @return {@code this} for chaining. - */ - @IgniteSpiConfiguration(optional = true) + @IgniteSpiConfiguration( + optional = true + ) public TcpDiscoveryS3IpFinder setSSEAlgorithm(String sseAlg) { this.sseAlg = sseAlg; - return this; } - /** - * Sets Amazon client configuration. - *

- * For details refer to Amazon S3 API reference. - * - * @param cfg Amazon client configuration. - * @return {@code this} for chaining. - */ - @IgniteSpiConfiguration(optional = true) + @IgniteSpiConfiguration( + optional = true + ) public TcpDiscoveryS3IpFinder setClientConfiguration(ClientConfiguration cfg) { this.cfg = cfg; - return this; } - /** - * Sets AWS credentials. Either use {@link #setAwsCredentialsProvider(AWSCredentialsProvider)} or this one. - *

- * For details refer to Amazon S3 API reference. - * - * @param cred AWS credentials. - * @return {@code this} for chaining. - */ - @IgniteSpiConfiguration(optional = false) + @IgniteSpiConfiguration( + optional = false + ) public TcpDiscoveryS3IpFinder setAwsCredentials(AWSCredentials cred) { this.cred = cred; - return this; } - /** - * Sets AWS credentials provider. Either use {@link #setAwsCredentials(AWSCredentials)} or this one. - *

- * For details refer to Amazon S3 API reference. - * - * @param credProvider AWS credentials provider. - * @return {@code this} for chaining. - */ - @IgniteSpiConfiguration(optional = false) + @IgniteSpiConfiguration( + optional = false + ) public TcpDiscoveryS3IpFinder setAwsCredentialsProvider(AWSCredentialsProvider credProvider) { this.credProvider = credProvider; - return this; } - /** {@inheritDoc} */ - @Override public TcpDiscoveryS3IpFinder setShared(boolean shared) { + public TcpDiscoveryS3IpFinder setShared(boolean shared) { super.setShared(shared); - return this; } - /** {@inheritDoc} */ - @Override public String toString() { + public String toString() { return S.toString(TcpDiscoveryS3IpFinder.class, this, "super", super.toString()); } -} \ No newline at end of file +} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/modules/consul/licenses/apache-2.0.txt b/modules/consul/licenses/apache-2.0.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/modules/consul/licenses/apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/modules/consul/pom.xml b/modules/consul/pom.xml new file mode 100644 index 0000000000000..8ecb57e4dd3d0 --- /dev/null +++ b/modules/consul/pom.xml @@ -0,0 +1,117 @@ + + + 4.0.0 + + + org.apache.ignite + apache-ignite + 2.5.0-SNAPSHOT + ../.. + + + ignite-consul + 2.5.0-SNAPSHOT + http://ignite.apache.org + + + org.apache.ignite + ignite-core + ${project.version} + + + + com.fasterxml.jackson.core + jackson-core + ${jackson2.version} + + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson2.version} + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson2.version} + + + + joda-time + joda-time + 2.8.1 + + + + org.apache.httpcomponents + httpclient + ${httpclient.version} + + + + org.apache.httpcomponents + httpcore + ${httpcore.version} + + + + commons-codec + commons-codec + ${commons.codec.version} + + + + org.apache.ignite + ignite-core + ${project.version} + test-jar + test + + + + org.springframework + spring-beans + ${spring.version} + test + + + + log4j + log4j + test + + + + org.springframework + spring-context + ${spring.version} + test + + + + org.springframework + spring-core + ${spring.version} + test + + + + com.ecwid.consul + consul-api + 1.3.1 + + + + + + + org.apache.felix + maven-bundle-plugin + + + + + \ No newline at end of file diff --git a/modules/consul/src/main/java/org/apache/ignite/configuration/ConsulAddressResolver.java b/modules/consul/src/main/java/org/apache/ignite/configuration/ConsulAddressResolver.java new file mode 100644 index 0000000000000..a093c8c63b41d --- /dev/null +++ b/modules/consul/src/main/java/org/apache/ignite/configuration/ConsulAddressResolver.java @@ -0,0 +1,86 @@ +package org.apache.ignite.configuration; + +import com.ecwid.consul.v1.ConsulClient; +import com.ecwid.consul.v1.Response; +import com.ecwid.consul.v1.kv.model.GetValue; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.util.typedef.internal.SB; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.IgniteSpiException; + +import java.net.InetSocketAddress; +import java.util.*; + +public class ConsulAddressResolver implements AddressResolver { + public static final String DELIM = "."; + ConsulClient consulClient; + String hostUrl; + String serviceName = "ignite"; + boolean doCache = true; + private Map inetSockAddrMap = new HashMap<>(); + /** + * Grid logger. + */ + @LoggerResource + private IgniteLogger log; + + public ConsulAddressResolver(String hostUrl, String serviceName, boolean doCache) { + this.hostUrl = hostUrl; + this.serviceName = serviceName; + this.consulClient = new ConsulClient(this.hostUrl); + this.doCache = doCache; + } + + public ConsulAddressResolver(String hostUrl) { + this(hostUrl, "ignite", true); + } + + public ConsulAddressResolver(String hostUrl, String serviceName) { + this(hostUrl, serviceName, true); + } + + private InetSocketAddress getResolvedAddress(InetSocketAddress inetSocketAddress) { + InetSocketAddress mapAddress = null; + if (doCache) { + mapAddress = inetSockAddrMap.get(inetSocketAddress); + } + + if (!Optional.of(mapAddress).isPresent()) { + try { + Response value = consulClient.getKVValue(key(inetSocketAddress)); + String ipAddress = value.getValue().getValue(); + mapAddress = InetSocketAddress.createUnresolved(ipAddress, inetSocketAddress.getPort()); + if (doCache) + inetSockAddrMap.put(inetSocketAddress, mapAddress); + } catch (Exception e) { + throw new IgniteSpiException("get Value from consul service : ", e); + } + } + return mapAddress; + } + + @Override + public Collection getExternalAddresses(InetSocketAddress inetSocketAddress) { + return Collections.singleton(getResolvedAddress(inetSocketAddress)); + } + + /** + * Gets key for provided address. + * + * @param addr Node address. + * @return Key. + */ + private String key(InetSocketAddress addr) { + assert addr != null; + + SB sb = new SB(); + + sb.a(this.serviceName) + .a(DELIM).a(addr.getAddress() + .getHostAddress()) + .a(DELIM) + .a(addr.getPort()); + + return sb.toString(); + } +} diff --git a/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinder.java b/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinder.java new file mode 100644 index 0000000000000..c5380556c97ca --- /dev/null +++ b/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinder.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ignite.spi.discovery.tcp.ipfinder.consul; + +import com.ecwid.consul.v1.ConsulClient; +import com.ecwid.consul.v1.Response; +import com.ecwid.consul.v1.kv.model.GetValue; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.IgniteInterruptedCheckedException; +import org.apache.ignite.internal.util.tostring.GridToStringExclude; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.S; +import org.apache.ignite.internal.util.typedef.internal.SB; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.resources.LoggerResource; +import org.apache.ignite.spi.IgniteSpiConfiguration; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter; +import org.apache.ignite.spi.discovery.tcp.ipfinder.consul.util.PublicIpService; + +import java.net.InetSocketAddress; +import java.util.*; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Consul-based IP finder. + *

+ * For information about Concul service visit consul.io. + *

Configuration

+ *

Mandatory

+ *
    + *
  • Consult host url
    • + *
+ *

Optional

+ *
    + *
  • Shared flag
    • + *
  • PublicIpMapRequired flag
    • + *
+ *

+ * The finder will create S3 bucket with configured name. The bucket will contain entries named + * like the following: {@code 192.168.1.136#1001}. + *

+ * Note that storing data in Concul service. + * Choose another implementation of {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder} for local + * or home network tests. + *

+ * Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}. + */ +public class TcpDiscoveryConsulIpFinder extends TcpDiscoveryIpFinderAdapter { + /** + * Delimiter to use in entries name. + */ + public static final String DELIM = "."; + /** + * Init guard. + */ + @GridToStringExclude + private final AtomicBoolean initGuard = new AtomicBoolean(); + /** + * Init latch. + */ + @GridToStringExclude + private final CountDownLatch initLatch = new CountDownLatch(1); + /** + * Client to interact with S3 storage. + */ + @GridToStringExclude + ConsulClient consulClient; + String hostUrl; + String serviceName = "ignite"; + /** + * Grid logger. + */ + @LoggerResource + private IgniteLogger log; + @GridToStringExclude + private boolean isPublicIpMapRequired = false; + + /** + * Constructor. + */ + public TcpDiscoveryConsulIpFinder() { + setShared(true); + } + + /** + * {@inheritDoc} + */ + @Override + public Collection getRegisteredAddresses() throws IgniteSpiException { + initClient(); + + Collection addrs = new LinkedList<>(); + + try { + Response> list = consulClient.getKVValues(this.serviceName); + + for (GetValue sum : list.getValue()) { + String key = sum.getKey(); + + StringTokenizer st = new StringTokenizer(key, DELIM); + + if (st.countTokens() != 3) + U.error(log, "Failed to parse S3 entry due to invalid format: " + key); + else { + String serviceName = st.nextToken(); + String addrStr = st.nextToken(); + String portStr = st.nextToken(); + + int port = -1; + + try { + port = Integer.parseInt(portStr); + } catch (NumberFormatException e) { + U.error(log, "Failed to parse port for S3 entry: " + key, e); + } + + if (port != -1) + try { + addrs.add(new InetSocketAddress(addrStr, port)); + } catch (IllegalArgumentException e) { + U.error(log, "Failed to parse port for S3 entry: " + key, e); + } + } + } + } catch (Exception e) { + throw new IgniteSpiException("Failed to list objects in the bucket: ", e); + } + + return addrs; + } + + /** + * {@inheritDoc} + */ + @Override + public void registerAddresses(Collection addrs) throws IgniteSpiException { + assert !F.isEmpty(addrs); + + initClient(); + + for (InetSocketAddress addr : addrs) { + String key = key(addr); + try { + String communicationIp = addr.getAddress().getHostAddress(); + if (isPublicIpMapRequired) { + communicationIp = PublicIpService.getPublicIp(); + } + consulClient.setKVValue(key, communicationIp.trim()); + } catch (Exception e) { + throw new IgniteSpiException("Failed to put entry [entry=" + key + ']', e); + } + } + } + + /** + * {@inheritDoc} + */ + @Override + public void unregisterAddresses(Collection addrs) throws IgniteSpiException { + assert !F.isEmpty(addrs); + + initClient(); + + for (InetSocketAddress addr : addrs) { + String key = key(addr); + + try { + consulClient.deleteKVValue(key); + } catch (Exception e) { + throw new IgniteSpiException("Failed to delete entry [" + key + ']', e); + } + } + } + + /** + * Gets key for provided address. + * + * @param addr Node address. + * @return Key. + */ + private String key(InetSocketAddress addr) { + assert addr != null; + + SB sb = new SB(); + + sb.a(this.serviceName) + .a(DELIM).a(addr.getAddress() + .getHostAddress()) + .a(DELIM) + .a(addr.getPort()); + + return sb.toString(); + } + + /** + * ConsulClient initialization. + * + * @throws org.apache.ignite.spi.IgniteSpiException In case of error. + */ + @SuppressWarnings({"BusyWait"}) + private void initClient() throws IgniteSpiException { + if (initGuard.compareAndSet(false, true)) + try { + if (hostUrl == null) + throw new IgniteSpiException("Consul HostURL not set."); + + if (consulClient == null) + U.warn(log, "Consul consulClient configuration is not set (will use default)."); + + + consulClient = createConsulClient(); + this.isPublicIpMapRequired = Optional.of(this.isPublicIpMapRequired).orElse(false); + + } finally { + initLatch.countDown(); + } + else { + try { + U.await(initLatch); + } catch (IgniteInterruptedCheckedException e) { + throw new IgniteSpiException("Thread has been interrupted.", e); + } + + if (consulClient == null) + throw new IgniteSpiException("Ip finder has not been initialized properly."); + } + } + + /** + * Instantiates {@code ConsulClient} instance. + * + * @return Client instance to use to connect to Consul. + */ + private ConsulClient createConsulClient() { + return new ConsulClient(this.hostUrl); + } + + /** + * Sets hostUrl this one. + *

+ * For details refer to Consul Java API reference. + * + * @param hostUrl AWS credentials provider. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) + public TcpDiscoveryConsulIpFinder setHostUrl(String hostUrl) { + this.hostUrl = hostUrl; + return this; + } + + /** + * Sets hostUrl this one. + *

+ * For details refer to Consul Java API reference. + * + * @param isPublicIpMapRequired get public IP + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) + public TcpDiscoveryConsulIpFinder isPublicIpMapRequired(boolean isPublicIpMapRequired) { + this.isPublicIpMapRequired = isPublicIpMapRequired; + return this; + } + + /** + * Sets serviceName this one. + *

+ * For details refer to Consul Java API reference. + * + * @param serviceName AWS credentials provider. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = true) + public TcpDiscoveryConsulIpFinder setServiceName(String serviceName) { + this.serviceName = serviceName; + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public TcpDiscoveryConsulIpFinder setShared(boolean shared) { + super.setShared(shared); + return this; + } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return S.toString(TcpDiscoveryConsulIpFinder.class, this, "super", super.toString()); + } +} diff --git a/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java b/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java new file mode 100644 index 0000000000000..0abb78235ac51 --- /dev/null +++ b/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains AWS S3-based IP finder. + */ +package org.apache.ignite.spi.discovery.tcp.ipfinder.consul; \ No newline at end of file diff --git a/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/util/PublicIpService.java b/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/util/PublicIpService.java new file mode 100644 index 0000000000000..998db28eb2b14 --- /dev/null +++ b/modules/consul/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/util/PublicIpService.java @@ -0,0 +1,45 @@ +package org.apache.ignite.spi.discovery.tcp.ipfinder.consul.util; + +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; + +/** + * @author rjain + */ +public class PublicIpService { + private static final List publicIpServices = Arrays.asList("https://api.ipify.org");//,"https://ipapi.co/ip"); + private static HttpClient httpClient = HttpClientBuilder.create().build(); + + private PublicIpService() { + } + + /** + * Gets the public ip address through ipify's api. By default, uses + * an http connection (which is about twice as fast as an https connection). + * + * @return The public ip address. + * @throws IOException If there is an IO error. + */ + public static String getPublicIp() throws IOException { + return getPublicIp(false); + } + + /** + * Gets the public ip address through ipify's api. + * + * @param useHttps If true, will use an https connection. If false, will use http. + * @return The public ip address. + * @throws IOException If there is an IO error. + */ + public static String getPublicIp(boolean useHttps) throws IOException { + String hostUrl = publicIpServices.get(Math.min((int) (System.currentTimeMillis() % 10), publicIpServices.size() - 1)); + String ip = httpClient.execute(new HttpGet(hostUrl)).getEntity().getContent().toString(); + return ip; + } + +} \ No newline at end of file diff --git a/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderAbstractSelfTest.java b/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderAbstractSelfTest.java new file mode 100644 index 0000000000000..ed6240f3dde3f --- /dev/null +++ b/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderAbstractSelfTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.tcp.ipfinder.consul; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Collection; +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAbstractSelfTest; +import org.apache.ignite.testsuites.IgniteIgnore; +import org.apache.ignite.testsuites.IgniteConsulTestSuite; + +/** + * Abstract TcpDiscoveryConsulIpFinder to test with different ways of setting AWS credentials. + */ +abstract class TcpDiscoveryConsulIpFinderAbstractSelfTest + extends TcpDiscoveryIpFinderAbstractSelfTest { + + /** + * Constructor. + * + * @throws Exception If any error occurs. + */ + TcpDiscoveryConsulIpFinderAbstractSelfTest() throws Exception { + } + + /** {@inheritDoc} */ + @Override protected TcpDiscoveryConsulIpFinder ipFinder() throws Exception { + TcpDiscoveryConsulIpFinder finder = new TcpDiscoveryConsulIpFinder(); + + injectLogger(finder); + + assert finder.isShared() : "Ip finder should be shared by default."; + + + + for (int i = 0; i < 5; i++) { + Collection addrs = finder.getRegisteredAddresses(); + + if (!addrs.isEmpty()) + finder.unregisterAddresses(addrs); + else + return finder; + + U.sleep(1000); + } + + if (!finder.getRegisteredAddresses().isEmpty()) + throw new Exception("Failed to initialize IP finder."); + + return finder; + } + + /** {@inheritDoc} */ + @IgniteIgnore("https://issues.apache.org/jira/browse/IGNITE-2420") + @Override public void testIpFinder() throws Exception { + super.testIpFinder(); + } + + + protected void setHostUrl(TcpDiscoveryConsulIpFinder finder) { + finder.setHostUrl(getHostUrl()); + } + + /** + * Gets Bucket name. + * Bucket name should be unique for the host to parallel test run on one bucket. + * Please note that the final bucket name should not exceed 63 chars. + * + * @return Bucket name. + */ + static String getHostUrl() { + String hostUrl; + try { + hostUrl = IgniteConsulTestSuite.getHostUrl( + "ip-finder-unit-test-" + InetAddress.getLocalHost().getHostName().toLowerCase()); + } + catch (UnknownHostException e) { + hostUrl = IgniteConsulTestSuite.getHostUrl( + "ip-finder-unit-test-rnd-" + ThreadLocalRandom.current().nextInt(100)); + } + + return hostUrl; + } +} \ No newline at end of file diff --git a/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderHostUrlSelfTest.java b/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderHostUrlSelfTest.java new file mode 100644 index 0000000000000..f736612356d56 --- /dev/null +++ b/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/TcpDiscoveryConsulIpFinderHostUrlSelfTest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.tcp.ipfinder.consul; + +import org.apache.ignite.testsuites.IgniteConsulTestSuite; + +/** + * TcpDiscoveryConsulIpFinder test using Consul credentials. + */ +public class TcpDiscoveryConsulIpFinderHostUrlSelfTest extends TcpDiscoveryConsulIpFinderAbstractSelfTest { + /** + * Constructor. + * + * @throws Exception If any error occurs. + */ + public TcpDiscoveryConsulIpFinderHostUrlSelfTest() throws Exception { + // No-op. + } + + /** {@inheritDoc} */ + @Override protected void setHostUrl(TcpDiscoveryConsulIpFinder finder) { + finder.setHostUrl(getHostUrl()); + } + + /** {@inheritDoc} */ + @Override public void testIpFinder() throws Exception { + super.testIpFinder(); + } +} \ No newline at end of file diff --git a/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java b/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java new file mode 100644 index 0000000000000..a5f518492294b --- /dev/null +++ b/modules/consul/src/test/java/org/apache/ignite/spi/discovery/tcp/ipfinder/consul/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * + * Contains internal tests or test related classes and interfaces. + */ +package org.apache.ignite.spi.discovery.tcp.ipfinder.consul; \ No newline at end of file diff --git a/modules/consul/src/test/java/org/apache/ignite/testsuites/IgniteConsulTestSuite.java b/modules/consul/src/test/java/org/apache/ignite/testsuites/IgniteConsulTestSuite.java new file mode 100644 index 0000000000000..547eb168c2224 --- /dev/null +++ b/modules/consul/src/test/java/org/apache/ignite/testsuites/IgniteConsulTestSuite.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.testsuites; + +import junit.framework.TestSuite; +import org.apache.ignite.spi.discovery.tcp.ipfinder.consul.TcpDiscoveryConsulIpFinderHostUrlSelfTest; +import org.apache.ignite.testframework.IgniteTestSuite; + +//todo: Use Mock Testing framework +/** + * Consul integration tests. + */ +public class IgniteConsulTestSuite extends TestSuite { + /** + * @return Test suite. + * @throws Exception Thrown in case of the failure. + */ + public static TestSuite suite() throws Exception { + TestSuite suite = new IgniteTestSuite("Consul Integration Test Suite"); + + // Consul IP finder. + suite.addTestSuite(TcpDiscoveryConsulIpFinderHostUrlSelfTest.class); + return suite; + } + + public static String getHostUrl(final String defaultHostUrl) { + String value = getRequiredEnvVar("test.Consul.hosturl"); + + return value == null ? defaultHostUrl : value; + } + + private static String getRequiredEnvVar(String name) { + String key = System.getenv(name); + + assert key != null : String.format("Environment variable '%s' is not set", name); + + return key; + } + + +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index 086d241f829af..8f044ecdc27bc 100644 --- a/pom.xml +++ b/pom.xml @@ -98,6 +98,7 @@ modules/rocketmq modules/sqlline modules/ml + modules/consul From 3202e71257418320e823093940461b0c03973812 Mon Sep 17 00:00:00 2001 From: Rahul Date: Sat, 21 Apr 2018 13:11:53 -0700 Subject: [PATCH 02/27] AWS S3 IP Finder SPI , replace depricated AWS APIs with new API added support to include public IP (for public address resolution) Added new S3 Address resolution depends on update S3 IP Finders SPI Add Consul IP Finder SPI and address resolvers --- modules/consul/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/consul/pom.xml b/modules/consul/pom.xml index 8ecb57e4dd3d0..3246c932bf0ac 100644 --- a/modules/consul/pom.xml +++ b/modules/consul/pom.xml @@ -6,9 +6,9 @@ org.apache.ignite - apache-ignite - 2.5.0-SNAPSHOT - ../.. + ignite-parent + 1 + ../../parent ignite-consul From 3145ab10b12a176693d96e127211a24df61e08e9 Mon Sep 17 00:00:00 2001 From: Rahul Date: Tue, 2 Oct 2018 21:57:33 -0700 Subject: [PATCH 03/27] resolved merge conflicts --- .../ipfinder/s3/TcpDiscoveryS3IpFinder.java | 463 +++++++++--------- 1 file changed, 237 insertions(+), 226 deletions(-) diff --git a/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java b/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java index 2a453a2358053..08c91e5008626 100644 --- a/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java +++ b/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java @@ -21,19 +21,15 @@ import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.regions.Region; import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; import com.amazonaws.services.s3.model.ObjectListing; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; -import java.io.ByteArrayInputStream; -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.LinkedList; -import java.util.StringTokenizer; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.regex.Pattern; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.HttpClientBuilder; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.IgniteInterruptedCheckedException; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -46,15 +42,17 @@ import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinderAdapter; import org.jetbrains.annotations.Nullable; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.HttpClient; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.HttpClientBuilder; + +import java.io.ByteArrayInputStream; import java.io.IOException; -import java.util.*; +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.LinkedList; +import java.util.Optional; +import java.util.StringTokenizer; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; /** * AWS S3-based IP finder. @@ -63,17 +61,17 @@ *

Configuration

*

Mandatory

*
    - *
  • AWS credentials (see {@link #setAwsCredentials(AWSCredentials)} and - * {@link #setAwsCredentialsProvider(AWSCredentialsProvider)}
    • - *
  • Bucket name (see {@link #setBucketName(String)})
    • + *
  • AWS credentials (see {@link #setAwsCredentials(AWSCredentials)} and + * {@link #setAwsCredentialsProvider(AWSCredentialsProvider)}
    • + *
  • Bucket name (see {@link #setBucketName(String)})
    • *
*

Optional

*
    - *
  • Client configuration (see {@link #setClientConfiguration(ClientConfiguration)})
    • - *
  • Shared flag (see {@link #setShared(boolean)})
    • - *
  • Bucket endpoint (see {@link #setBucketEndpoint(String)})
    • - *
  • Region (see {@link #setRegion(String)})
    • - *
  • Server side encryption algorithm (see {@link #setSSEAlgorithm(String)})
    • + *
  • Client configuration (see {@link #setClientConfiguration(ClientConfiguration)})
    • + *
  • Shared flag (see {@link #setShared(boolean)})
    • + *
  • Bucket endpoint (see {@link #setBucketEndpoint(String)})
    • + *
  • Region (see {@link #setRegion(String)})
    • + *
  • Server side encryption algorithm (see {@link #setSSEAlgorithm(String)})
    • *
  • Server side encryption algorithm (see {@link #setKeyPrefix(String)})
    • *
*

@@ -86,23 +84,31 @@ *

* Note that this finder is shared by default (see {@link org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder#isShared()}. */ - public class TcpDiscoveryS3IpFinder extends TcpDiscoveryIpFinderAdapter { - private static final String DELIM = "#"; + /** + * Delimiter to use in S3 entries name. + */ + public static final String DELIM = "#"; + @GridToStringExclude private static HttpClient httpClient = HttpClientBuilder.create().build(); + @GridToStringExclude - private final ObjectMetadata objMetadata = new ObjectMetadata(); + private final byte[] DEFAULT_ENTRY_CONTENT = new byte[]{1}; + /** Entry metadata. */ @GridToStringExclude - private final AtomicBoolean initGuard = new AtomicBoolean(); + private final ObjectMetadata objMetadata = new ObjectMetadata(); + /** Init latch. */ @GridToStringExclude private final CountDownLatch initLatch = new CountDownLatch(1); - private byte[] entryContent = new byte[]{1}; + @GridToStringExclude + private byte[] entryContent = DEFAULT_ENTRY_CONTENT; + /** Grid logger. */ @LoggerResource private IgniteLogger log; + /** Client to interact with S3 storage. */ @GridToStringExclude private AmazonS3 s3; - private String bucketName; /** Bucket endpoint. */ @Nullable private String bucketEndpoint; @@ -112,33 +118,43 @@ public class TcpDiscoveryS3IpFinder extends TcpDiscoveryIpFinderAdapter { /** Sub-folder name to write node addresses. */ @Nullable private String keyPrefix; + /** + * Bucket name. + */ + private String bucketName; /** Init guard. */ @GridToStringExclude private final AtomicBoolean initGuard = new AtomicBoolean(); - - /** Init latch. */ + /** Do create public ip to private Map */ @GridToStringExclude private boolean isPublicIpMapRequired = false; - @Nullable - private String bucketEndpoint; - @Nullable - private String region; - @Nullable - private String sseAlg; + /** Amazon client configuration. */ private ClientConfiguration cfg; + + /** AWS Credentials. */ @GridToStringExclude private AWSCredentials cred; + + /** AWS Credentials. */ @GridToStringExclude private AWSCredentialsProvider credProvider; + /** + * Constructor. + */ public TcpDiscoveryS3IpFinder() { - this.setShared(true); + setShared(true); } + /** + * {@inheritDoc} + */ + @Override public Collection getRegisteredAddresses() throws IgniteSpiException { - this.initClient(); - LinkedList addrs = new LinkedList(); + initClient(); + + Collection addrs = new LinkedList<>(); try { ObjectListing list; @@ -149,10 +165,7 @@ public Collection getRegisteredAddresses() throws IgniteSpiEx list = s3.listObjects(bucketName, keyPrefix); while (true) { - Iterator var3 = list.getObjectSummaries().iterator(); - - while (var3.hasNext()) { - S3ObjectSummary sum = (S3ObjectSummary) var3.next(); + for (S3ObjectSummary sum : list.getObjectSummaries()) { String key = sum.getKey(); String addr = key; @@ -166,6 +179,7 @@ public Collection getRegisteredAddresses() throws IgniteSpiEx else { String addrStr = st.nextToken(); String portStr = st.nextToken(); + int port = -1; try { @@ -175,11 +189,9 @@ public Collection getRegisteredAddresses() throws IgniteSpiEx U.error(log, "Failed to parse port for S3 entry: " + addr, e); } - if (port != -1) { + if (port != -1) try { addrs.add(new InetSocketAddress(addrStr, port)); - } catch (IllegalArgumentException var11) { - U.error(this.log, "Failed to parse port for S3 entry: " + key, var11); } catch (IllegalArgumentException e) { U.error(log, "Failed to parse port for S3 entry: " + addr, e); @@ -187,67 +199,66 @@ public Collection getRegisteredAddresses() throws IgniteSpiEx } } - if (!list.isTruncated()) { - return addrs; - } - - list = this.s3.listNextBatchOfObjects(list); + if (list.isTruncated()) + list = s3.listNextBatchOfObjects(list); + else + break; } - } catch (AmazonClientException var13) { - throw new IgniteSpiException("Failed to list objects in the bucket: " + this.bucketName, var13); + } catch (AmazonClientException e) { + throw new IgniteSpiException("Failed to list objects in the bucket: " + bucketName, e); } + + return addrs; } + /** + * {@inheritDoc} + */ + @Override public void registerAddresses(Collection addrs) throws IgniteSpiException { assert !F.isEmpty(addrs); - this.initClient(); - Iterator var2 = addrs.iterator(); + initClient(); + + for (InetSocketAddress addr : addrs) { + String key = key(addr); - while (var2.hasNext()) { - InetSocketAddress addr = (InetSocketAddress) var2.next(); - String key = this.key(addr); try { - this.s3.putObject(this.bucketName, key, new ByteArrayInputStream(getEntryContents()), this.objMetadata); - } catch (AmazonClientException var6) { - throw new IgniteSpiException("Failed to put entry [bucketName=" + this.bucketName + ", entry=" + key + ']', var6); + s3.putObject(bucketName, key, new ByteArrayInputStream(getEntryContents()), objMetadata); + } catch (AmazonClientException e) { + throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName + + ", entry=" + key + ']', e); } } - } + /** + * {@inheritDoc} + */ + @Override public void unregisterAddresses(Collection addrs) throws IgniteSpiException { assert !F.isEmpty(addrs); - this.initClient(); - Iterator var2 = addrs.iterator(); + initClient(); - while (var2.hasNext()) { - InetSocketAddress addr = (InetSocketAddress) var2.next(); - String key = this.key(addr); + for (InetSocketAddress addr : addrs) { + String key = key(addr); try { - this.s3.deleteObject(this.bucketName, key); - } catch (AmazonClientException var6) { - throw new IgniteSpiException("Failed to delete entry [bucketName=" + this.bucketName + ", entry=" + key + ']', var6); - } - } - - } - - private byte[] getEntryContents() { - try { - if (isPublicIpMapRequired) { - entryContent = httpClient.execute(new HttpGet("http://169.254.169.254/latest/meta-data/public-ipv4")).getEntity().getContent().toString().getBytes(); + s3.deleteObject(bucketName, key); + } catch (AmazonClientException e) { + throw new IgniteSpiException("Failed to delete entry [bucketName=" + bucketName + + ", entry=" + key + ']', e); } - return entryContent; - } catch (ClientProtocolException e) { - throw new IgniteSpiException("Failed to get public IP", e); - } catch (IOException e) { - throw new IgniteSpiException("Failed to get public IP", e); } } + /** + * Gets S3 key for provided address. + * + * @param addr Node address. + * @return Key. + */ private String key(InetSocketAddress addr) { assert addr != null; @@ -259,170 +270,98 @@ private String key(InetSocketAddress addr) { sb.a(keyPrefix); sb.a(addrStr) - .a(DELIM) - .a(addr.getPort()); + .a(DELIM) + .a(addr.getPort()); return sb.toString(); } + /** + * Amazon s3 client initialization. + * + * @throws org.apache.ignite.spi.IgniteSpiException In case of error. + */ + @SuppressWarnings({"BusyWait"}) private void initClient() throws IgniteSpiException { - if (this.initGuard.compareAndSet(false, true)) { + if (initGuard.compareAndSet(false, true)) try { - if (this.cred == null && this.credProvider == null) { + if (cred == null && credProvider == null) throw new IgniteSpiException("AWS credentials are not set."); - } - if (this.cfg == null) { - U.warn(this.log, "Amazon client configuration is not set (will use default)."); - } + if (cfg == null) + U.warn(log, "Amazon client configuration is not set (will use default)."); - if (F.isEmpty(this.bucketName)) { + if (F.isEmpty(bucketName)) throw new IgniteSpiException("Bucket name is null or empty (provide bucket name and restart)."); - } this.isPublicIpMapRequired = Optional.of(this.isPublicIpMapRequired).orElse(false); this.objMetadata.setContentLength((long) getEntryContents().length); - if (!F.isEmpty(this.sseAlg)) { - this.objMetadata.setSSEAlgorithm(this.sseAlg); - } - this.s3 = this.createAmazonS3Client(); - if (!this.s3.doesBucketExist(this.bucketName)) { + if (!F.isEmpty(sseAlg)) + objMetadata.setSSEAlgorithm(sseAlg); + + s3 = createAmazonS3Client(); + + if (!s3.doesBucketExist(bucketName)) { try { - this.s3.createBucket(this.bucketName); - if (this.log.isDebugEnabled()) { - this.log.debug("Created S3 bucket: " + this.bucketName); - } + s3.createBucket(bucketName); + + if (log.isDebugEnabled()) + log.debug("Created S3 bucket: " + bucketName); - while (!this.s3.doesBucketExist(this.bucketName)) { + while (!s3.doesBucketExist(bucketName)) try { - U.sleep(200L); - } catch (IgniteInterruptedCheckedException var8) { - throw new IgniteSpiException("Thread has been interrupted.", var8); + U.sleep(200); + } catch (IgniteInterruptedCheckedException e) { + throw new IgniteSpiException("Thread has been interrupted.", e); } - } - } catch (AmazonClientException var9) { - if (!this.s3.doesBucketExist(this.bucketName)) { - this.s3 = null; - throw new IgniteSpiException("Failed to create bucket: " + this.bucketName, var9); + } catch (AmazonClientException e) { + if (!s3.doesBucketExist(bucketName)) { + s3 = null; + + throw new IgniteSpiException("Failed to create bucket: " + bucketName, e); } } } } finally { - this.initLatch.countDown(); + initLatch.countDown(); } - } else { + else { try { - U.await(this.initLatch); - } catch (IgniteInterruptedCheckedException var7) { - throw new IgniteSpiException("Thread has been interrupted.", var7); + U.await(initLatch); + } catch (IgniteInterruptedCheckedException e) { + throw new IgniteSpiException("Thread has been interrupted.", e); } - if (this.s3 == null) { + if (s3 == null) throw new IgniteSpiException("Ip finder has not been initialized properly."); - } - } - -// /** -// * Instantiates {@code AmazonS3Client} instance. -// * -// * @return Client instance to use to connect to AWS. -// */ -// AmazonS3Client createAmazonS3Client() { -// AmazonS3Client cln = cfg != null -// ? (cred != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(credProvider, cfg)) -// : (cred != null ? new AmazonS3Client(cred) : new AmazonS3Client(credProvider)); -// } - - private AmazonS3 createAmazonS3Client() { - if (this.credProvider == null) { - this.credProvider = new AWSStaticCredentialsProvider(this.cred); - } - AmazonS3 s3Client = AmazonS3ClientBuilder.standard() - .withRegion(region) - .withCredentials(credProvider) - .build(); - if (!F.isEmpty(this.bucketEndpoint)) { - s3Client.setEndpoint(this.bucketEndpoint); - } - - if (!F.isEmpty(this.region)) { - s3Client.setRegion(Region.getRegion(Regions.fromName(this.region))); - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - } - - - return s3Client; } - - @IgniteSpiConfiguration( - optional = true - ) - public TcpDiscoveryS3IpFinder setRegion(String region) { - this.region = region; - return this; + /** + * Instantiates {@code AmazonS3Client} instance. + * + * @return Client instance to use to connect to AWS. + */ + AmazonS3Client createAmazonS3Client() { + AmazonS3Client cln = cfg != null + ? (cred != null ? new AmazonS3Client(cred, cfg) : new AmazonS3Client(credProvider, cfg)) + : (cred != null ? new AmazonS3Client(cred) : new AmazonS3Client(credProvider)); + return cln; } - @IgniteSpiConfiguration( - optional = true - ) - public TcpDiscoveryS3IpFinder setPublicIpMapRequired(boolean publicIpMapRequired) { - isPublicIpMapRequired = publicIpMapRequired; - return this; - } - - @IgniteSpiConfiguration( - optional = false - ) + /** + * Sets bucket name for IP finder. + * + * @param bucketName Bucket name. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) public TcpDiscoveryS3IpFinder setBucketName(String bucketName) { this.bucketName = bucketName; + return this; } @@ -437,6 +376,7 @@ public TcpDiscoveryS3IpFinder setBucketName(String bucketName) { @IgniteSpiConfiguration(optional = true) public TcpDiscoveryS3IpFinder setBucketEndpoint(String bucketEndpoint) { this.bucketEndpoint = bucketEndpoint; + return this; } @@ -451,40 +391,111 @@ public TcpDiscoveryS3IpFinder setBucketEndpoint(String bucketEndpoint) { @IgniteSpiConfiguration(optional = true) public TcpDiscoveryS3IpFinder setSSEAlgorithm(String sseAlg) { this.sseAlg = sseAlg; + return this; } - @IgniteSpiConfiguration( - optional = true - ) + /** + * Sets Amazon client configuration. + *

+ * For details refer to Amazon S3 API reference. + * + * @param cfg Amazon client configuration. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = true) public TcpDiscoveryS3IpFinder setClientConfiguration(ClientConfiguration cfg) { this.cfg = cfg; + return this; } - @IgniteSpiConfiguration( - optional = false - ) + /** + * Sets AWS credentials. Either use {@link #setAwsCredentialsProvider(AWSCredentialsProvider)} or this one. + *

+ * For details refer to Amazon S3 API reference. + * + * @param cred AWS credentials. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) public TcpDiscoveryS3IpFinder setAwsCredentials(AWSCredentials cred) { this.cred = cred; + return this; } - @IgniteSpiConfiguration( - optional = false - ) + /** + * Sets AWS credentials provider. Either use {@link #setAwsCredentials(AWSCredentials)} or this one. + *

+ * For details refer to Amazon S3 API reference. + * + * @param credProvider AWS credentials provider. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = false) public TcpDiscoveryS3IpFinder setAwsCredentialsProvider(AWSCredentialsProvider credProvider) { this.credProvider = credProvider; return this; } - public TcpDiscoveryS3IpFinder setShared(boolean shared) { + /** + * This can be thought of as the sub-folder within the bucket that will hold the node addresses. + *

+ * For details visit + * + * + * @param keyPrefix AWS credentials provider. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = true) + public TcpDiscoveryS3IpFinder setKeyPrefix(String keyPrefix) { + this.keyPrefix = keyPrefix; + + return this; + } + + /** + * This can be thought of as the private ip to public ip map . + *

+ * In case need to access out side of vpc/aws network need public ip to interact with the node you much use S3 address resolver {@link org.apache.ignite.configuration.S3AddressResolver) + * + * + * @param publicIpMapRequired AWS credentials provider. + * @return {@code this} for chaining. + */ + @IgniteSpiConfiguration(optional = true) + public TcpDiscoveryS3IpFinder setPublicIpMapRequired(boolean publicIpMapRequired) { + isPublicIpMapRequired = publicIpMapRequired; + return this; + } + + /** + * {@inheritDoc} + */ + @Override public TcpDiscoveryS3IpFinder setShared(boolean shared) { super.setShared(shared); + return this; } - public String toString() { + + /** {@inheritDoc} */ + @Override public String toString() { return S.toString(TcpDiscoveryS3IpFinder.class, this, "super", super.toString()); } + + private byte[] getEntryContents() { + try { + if (isPublicIpMapRequired && DEFAULT_ENTRY_CONTENT.equals(entryContent)) { + this.entryContent = httpClient.execute(new HttpGet("http://169.254.169.254/latest/meta-data/public-ipv4")).getEntity().getContent().toString().getBytes(); + } + return entryContent; + } catch (ClientProtocolException e) { + throw new IgniteSpiException("Failed to get public IP", e); + } catch (IOException e) { + throw new IgniteSpiException("Failed to get public IP", e); + } + } } \ No newline at end of file From 71698ded18e80257e1ea10f8b90665f06b846dcd Mon Sep 17 00:00:00 2001 From: Rahul Date: Tue, 2 Oct 2018 22:10:53 -0700 Subject: [PATCH 04/27] remove documentation error --- .../spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java b/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java index 08c91e5008626..4145a7c244ab0 100644 --- a/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java +++ b/modules/aws/src/main/java/org/apache/ignite/spi/discovery/tcp/ipfinder/s3/TcpDiscoveryS3IpFinder.java @@ -70,7 +70,6 @@ *

  • Client configuration (see {@link #setClientConfiguration(ClientConfiguration)})
    • *
  • Shared flag (see {@link #setShared(boolean)})
    • *
  • Bucket endpoint (see {@link #setBucketEndpoint(String)})
    • - *
  • Region (see {@link #setRegion(String)})
    • *
  • Server side encryption algorithm (see {@link #setSSEAlgorithm(String)})
    • *
  • Server side encryption algorithm (see {@link #setKeyPrefix(String)})
    • * From 739f50ea7f30556de9e890742738b7f204a4374a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:31 +0000 Subject: [PATCH 05/27] Bump xstream from 1.4.8 to 1.4.10-java7 in /modules/direct-io Bumps [xstream](https://github.com/x-stream/xstream) from 1.4.8 to 1.4.10-java7. - [Release notes](https://github.com/x-stream/xstream/releases) - [Commits](https://github.com/x-stream/xstream/commits) Signed-off-by: dependabot[bot] --- modules/direct-io/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml index e460e672f71f3..10900d339ab3c 100644 --- a/modules/direct-io/pom.xml +++ b/modules/direct-io/pom.xml @@ -94,7 +94,7 @@ com.thoughtworks.xstream xstream - 1.4.8 + 1.4.10-java7 test From 1a6e966bff2ac5f4e734ef0eae83687c44b2291d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:32 +0000 Subject: [PATCH 06/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/direct-io Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-indexing` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/direct-io/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml index e460e672f71f3..bb5d968407bae 100644 --- a/modules/direct-io/pom.xml +++ b/modules/direct-io/pom.xml @@ -32,7 +32,7 @@ ignite-direct-io - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 5b3e516d1f61a435e275ed449a4daf582d1652a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:32 +0000 Subject: [PATCH 07/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/dev-utils Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-indexing` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/dev-utils/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/dev-utils/pom.xml b/modules/dev-utils/pom.xml index afb2433260291..64eba7da13293 100644 --- a/modules/dev-utils/pom.xml +++ b/modules/dev-utils/pom.xml @@ -31,7 +31,7 @@ ignite-dev-utils - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From cca2fc4ae17dc2bb45ace1b6d643c64285853f7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:33 +0000 Subject: [PATCH 08/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/flink Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-log4j` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-spring` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/flink/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/flink/pom.xml b/modules/flink/pom.xml index c3a672a20532a..8c88d063cb002 100644 --- a/modules/flink/pom.xml +++ b/modules/flink/pom.xml @@ -31,7 +31,7 @@ ignite-flink - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From a3091885b85e222bdf36d4840020c6d145dfde22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:34 +0000 Subject: [PATCH 09/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/flume Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-spring` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-log4j` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/flume/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/flume/pom.xml b/modules/flume/pom.xml index 583d38ea5e1bf..cd4eca55f7b8a 100644 --- a/modules/flume/pom.xml +++ b/modules/flume/pom.xml @@ -31,7 +31,7 @@ ignite-flume - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 214986a3a7b6888bd4dea18e240abf17ecfc1b84 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:19:12 +0000 Subject: [PATCH 10/27] Bump xstream from 1.4.8 to 1.4.10-java7 in /modules/compatibility Bumps [xstream](https://github.com/x-stream/xstream) from 1.4.8 to 1.4.10-java7. - [Release notes](https://github.com/x-stream/xstream/releases) - [Commits](https://github.com/x-stream/xstream/commits) Signed-off-by: dependabot[bot] --- modules/compatibility/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml index 7f24efed3a5cb..4b27aba06e834 100644 --- a/modules/compatibility/pom.xml +++ b/modules/compatibility/pom.xml @@ -64,7 +64,7 @@ com.thoughtworks.xstream xstream - 1.4.8 + 1.4.10-java7 test From 8168e3d4a04e0acdd5c22dfdbe345aa91cb62c72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:19:56 +0000 Subject: [PATCH 11/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/codegen Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-indexing` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/codegen/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/codegen/pom.xml b/modules/codegen/pom.xml index dd8dcbf1b3591..95fe0e0587c5c 100644 --- a/modules/codegen/pom.xml +++ b/modules/codegen/pom.xml @@ -32,7 +32,7 @@ ignite-codegen - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 969ec989495972a4d2174d28f3b4f4c4959b57f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:19:58 +0000 Subject: [PATCH 12/27] Bump ignite-core from 2.7.0-SNAPSHOT to 2.8.1 in /modules/compatibility Bumps ignite-core from 2.7.0-SNAPSHOT to 2.8.1. Signed-off-by: dependabot[bot] --- modules/compatibility/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml index 4b27aba06e834..9c1f3011defd1 100644 --- a/modules/compatibility/pom.xml +++ b/modules/compatibility/pom.xml @@ -33,7 +33,7 @@ ignite-compatibility - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From cf1f7f8d55c0e0f693649c03495cadb4117f3b86 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:20:15 +0000 Subject: [PATCH 13/27] Bump ignite-core from 2.7.0-SNAPSHOT to 2.8.1 in /modules/cloud Bumps ignite-core from 2.7.0-SNAPSHOT to 2.8.1. Signed-off-by: dependabot[bot] --- modules/cloud/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/cloud/pom.xml b/modules/cloud/pom.xml index 482500e96f81e..0420082e25590 100644 --- a/modules/cloud/pom.xml +++ b/modules/cloud/pom.xml @@ -29,7 +29,7 @@ ignite-cloud - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 01c38a231059e61bc65cbcd12b7c0e42c74eceab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:20:49 +0000 Subject: [PATCH 14/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/clients Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-spring` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-indexing` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-log4j` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-rest-http` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-tools` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/clients/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml index 621582619e54c..e7409323b7433 100644 --- a/modules/clients/pom.xml +++ b/modules/clients/pom.xml @@ -31,7 +31,7 @@ ignite-clients - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From f1dc391a6385260b739af3e55da8d5af9a127639 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:22:09 +0000 Subject: [PATCH 15/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/camel Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-log4j` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-spring` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/camel/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/camel/pom.xml b/modules/camel/pom.xml index 3942c9bb1dcd5..6df9d8e8478ba 100644 --- a/modules/camel/pom.xml +++ b/modules/camel/pom.xml @@ -31,7 +31,7 @@ ignite-camel - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 8e3abcef72aee72a79c182668626c819951b0fa0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:22:29 +0000 Subject: [PATCH 16/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/benchmarks Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-indexing` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/benchmarks/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/benchmarks/pom.xml b/modules/benchmarks/pom.xml index 1ea984c6c544e..bb407a10d6962 100644 --- a/modules/benchmarks/pom.xml +++ b/modules/benchmarks/pom.xml @@ -31,7 +31,7 @@ ignite-benchmarks - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From d963c53cb555ae6ecaa2227007801dc7e857f00f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:22:39 +0000 Subject: [PATCH 17/27] Bump ignite-core from 2.7.0-SNAPSHOT to 2.8.1 in /modules/aws Bumps ignite-core from 2.7.0-SNAPSHOT to 2.8.1. Signed-off-by: dependabot[bot] --- modules/aws/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aws/pom.xml b/modules/aws/pom.xml index 57d77ffb96d3c..a2838003d4fce 100644 --- a/modules/aws/pom.xml +++ b/modules/aws/pom.xml @@ -31,7 +31,7 @@ ignite-aws - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 68728d33bafff39926196635de95917f8bd0b2b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 02:23:05 +0000 Subject: [PATCH 18/27] Bump project.version from 2.7.0-SNAPSHOT to 2.8.1 in /modules/aop Bumps `project.version` from 2.7.0-SNAPSHOT to 2.8.1. Updates `ignite-core` from 2.7.0-SNAPSHOT to 2.8.1 Updates `ignite-log4j` from 2.7.0-SNAPSHOT to 2.8.1 Signed-off-by: dependabot[bot] --- modules/aop/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aop/pom.xml b/modules/aop/pom.xml index 84300c1cfbf3f..b085c2bd4e675 100644 --- a/modules/aop/pom.xml +++ b/modules/aop/pom.xml @@ -31,7 +31,7 @@ ignite-aop - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From d06107f4247763ab2bad564312ef3855a789b414 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:31 +0000 Subject: [PATCH 19/27] Bump xstream from 1.4.8 to 1.4.10-java7 in /modules/core Bumps [xstream](https://github.com/x-stream/xstream) from 1.4.8 to 1.4.10-java7. - [Release notes](https://github.com/x-stream/xstream/releases) - [Commits](https://github.com/x-stream/xstream/commits) Signed-off-by: dependabot[bot] --- modules/core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/pom.xml b/modules/core/pom.xml index 9be521799f3a0..7bb4a02e39436 100644 --- a/modules/core/pom.xml +++ b/modules/core/pom.xml @@ -98,7 +98,7 @@ com.thoughtworks.xstream xstream - 1.4.8 + 1.4.10-java7 test From c5910a69477d2bbfc2024a7fa34a8db87101a753 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:31 +0000 Subject: [PATCH 20/27] Bump ignite-core from 2.7.0-SNAPSHOT to 2.8.1 in /modules/gce Bumps ignite-core from 2.7.0-SNAPSHOT to 2.8.1. Signed-off-by: dependabot[bot] --- modules/gce/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/gce/pom.xml b/modules/gce/pom.xml index 7b66c2ac20561..b682b969bf349 100644 --- a/modules/gce/pom.xml +++ b/modules/gce/pom.xml @@ -31,7 +31,7 @@ ignite-gce - 2.7.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 4d7f420155145569ccec982be13041f75552916d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2020 01:57:31 +0000 Subject: [PATCH 21/27] Bump ignite-core from 2.5.0-SNAPSHOT to 2.8.1 in /modules/consul Bumps ignite-core from 2.5.0-SNAPSHOT to 2.8.1. Signed-off-by: dependabot[bot] --- modules/consul/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/consul/pom.xml b/modules/consul/pom.xml index 3246c932bf0ac..728aa6d184db7 100644 --- a/modules/consul/pom.xml +++ b/modules/consul/pom.xml @@ -12,7 +12,7 @@ ignite-consul - 2.5.0-SNAPSHOT + 2.8.1 http://ignite.apache.org From 9e779aa165854ab71c6e90f3e8b310aaced74c6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2020 00:08:09 +0000 Subject: [PATCH 22/27] Bump junit from 4.11 to 4.13.1 in /parent Bumps [junit](https://github.com/junit-team/junit4) from 4.11 to 4.13.1. - [Release notes](https://github.com/junit-team/junit4/releases) - [Changelog](https://github.com/junit-team/junit4/blob/main/doc/ReleaseNotes4.11.md) - [Commits](https://github.com/junit-team/junit4/compare/r4.11...r4.13.1) Signed-off-by: dependabot[bot] --- parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parent/pom.xml b/parent/pom.xml index 412e81ed1521a..8f88c2da5f6a7 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -243,7 +243,7 @@ junit junit - 4.11 + 4.13.1 test From 21e220c9ab4b611f2db53238d69cdd47601267d2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Apr 2021 20:33:11 +0000 Subject: [PATCH 23/27] Bump commons-io from 2.4 to 2.7 in /modules/core Bumps commons-io from 2.4 to 2.7. Signed-off-by: dependabot[bot] --- modules/core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/pom.xml b/modules/core/pom.xml index 7bb4a02e39436..fee81ebdff09f 100644 --- a/modules/core/pom.xml +++ b/modules/core/pom.xml @@ -187,7 +187,7 @@ commons-io commons-io - 2.4 + 2.7 test From 2f3391a240dd12d61e4a5d81f69448ce9372b707 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Apr 2021 20:33:13 +0000 Subject: [PATCH 24/27] Bump commons-io from 2.4 to 2.7 in /modules/clients Bumps commons-io from 2.4 to 2.7. Signed-off-by: dependabot[bot] --- modules/clients/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/clients/pom.xml b/modules/clients/pom.xml index e7409323b7433..873bedbf2e46d 100644 --- a/modules/clients/pom.xml +++ b/modules/clients/pom.xml @@ -94,7 +94,7 @@ commons-io commons-io - 2.4 + 2.7 test From d8afe2bfa2290424be8d7689779d1331760bbf1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 20:38:51 +0000 Subject: [PATCH 25/27] Bump xstream from 1.4.10-java7 to 1.4.17 in /modules/compatibility Bumps [xstream](https://github.com/x-stream/xstream) from 1.4.10-java7 to 1.4.17. - [Release notes](https://github.com/x-stream/xstream/releases) - [Commits](https://github.com/x-stream/xstream/commits) Signed-off-by: dependabot[bot] --- modules/compatibility/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/compatibility/pom.xml b/modules/compatibility/pom.xml index 9c1f3011defd1..a892343fa123e 100644 --- a/modules/compatibility/pom.xml +++ b/modules/compatibility/pom.xml @@ -64,7 +64,7 @@ com.thoughtworks.xstream xstream - 1.4.10-java7 + 1.4.17 test From 9d234e05665a65cc833b38c579993fcd1faab3d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 20:38:51 +0000 Subject: [PATCH 26/27] Bump xstream from 1.4.10-java7 to 1.4.17 in /modules/core Bumps [xstream](https://github.com/x-stream/xstream) from 1.4.10-java7 to 1.4.17. - [Release notes](https://github.com/x-stream/xstream/releases) - [Commits](https://github.com/x-stream/xstream/commits) Signed-off-by: dependabot[bot] --- modules/core/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/core/pom.xml b/modules/core/pom.xml index 7bb4a02e39436..eefbff91aa42b 100644 --- a/modules/core/pom.xml +++ b/modules/core/pom.xml @@ -98,7 +98,7 @@ com.thoughtworks.xstream xstream - 1.4.10-java7 + 1.4.17 test From a3edc28b46a3dd2bcab6dcedb69810ba095cf74f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 May 2021 20:38:54 +0000 Subject: [PATCH 27/27] Bump xstream from 1.4.10-java7 to 1.4.17 in /modules/direct-io Bumps [xstream](https://github.com/x-stream/xstream) from 1.4.10-java7 to 1.4.17. - [Release notes](https://github.com/x-stream/xstream/releases) - [Commits](https://github.com/x-stream/xstream/commits) Signed-off-by: dependabot[bot] --- modules/direct-io/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/direct-io/pom.xml b/modules/direct-io/pom.xml index 84f800fcab979..36b5bc073f95b 100644 --- a/modules/direct-io/pom.xml +++ b/modules/direct-io/pom.xml @@ -94,7 +94,7 @@ com.thoughtworks.xstream xstream - 1.4.10-java7 + 1.4.17 test