ISPN-6847 Kubernetes PING integration

infinispan · Aug 2, 2016 · c6e5581 · c6e5581
1 parent 25f979f
commit c6e5581
Show file tree

Hide file tree

Showing 20 changed files with 344 additions and 1 deletion.
diff --git a/all/embedded/pom.xml b/all/embedded/pom.xml
@@ -32,6 +32,12 @@
          <optional>${uberjar.deps.optional}</optional>
       </dependency>
 
+      <dependency>
+         <groupId>org.infinispan</groupId>
+         <artifactId>infinispan-cloud</artifactId>
+         <optional>${uberjar.deps.optional}</optional>
+      </dependency>
+
       <dependency>
          <groupId>org.infinispan</groupId>
          <artifactId>infinispan-cachestore-jdbc</artifactId>
@@ -132,6 +138,22 @@
          <artifactId>c3p0</artifactId>
          <optional>${uberjar.deps.optional}</optional>
       </dependency>
+
+      <dependency>
+         <groupId>org.jboss</groupId>
+         <artifactId>jboss-dmr</artifactId>
+         <optional>${uberjar.deps.optional}</optional>
+      </dependency>
+      <dependency>
+         <groupId>org.jgroups.kubernetes</groupId>
+         <artifactId>dns</artifactId>
+         <optional>${uberjar.deps.optional}</optional>
+      </dependency>
+      <dependency>
+         <groupId>org.jgroups.kubernetes</groupId>
+         <artifactId>kubernetes</artifactId>
+         <optional>${uberjar.deps.optional}</optional>
+      </dependency>
    </dependencies>
 
    <build>
@@ -251,6 +273,14 @@
                      </filters>
                      <createSourcesJar>true</createSourcesJar>
                      <relocations>
+                        <relocation>
+                           <pattern>org.jboss.dmr</pattern>
+                           <shadedPattern>infinispan.org.jboss.dmr</shadedPattern>
+                        </relocation>
+                        <relocation>
+                           <pattern>net.oauth</pattern>
+                           <shadedPattern>infinispan.net.oauth</shadedPattern>
+                        </relocation>
                         <relocation>
                            <pattern>com.google</pattern>
                            <shadedPattern>infinispan.com.google</shadedPattern>
@@ -322,12 +352,16 @@
                            !org.jgroups.*,
                            !org.jboss.logmanager,
                            !org.jboss.modules,
+                           !org.jboss.dmr,
                            !org.iq80.snappy,
                            !org.jboss.marshalling.*,
                            !org.xerial.snappy,
                            !sun.misc,
                            !sun.reflect,
                            !sun.nio.ch,
+                           !__redirected,
+                           com.sun.net.httpserver;resolution:=optional,
+                           io.undertow.*;resolution:=optional,
                            *
                         </Import-Package>
                         <DynamicImport-Package>*</DynamicImport-Package>

diff --git a/as-modules/embedded/build.xml b/as-modules/embedded/build.xml
@@ -161,6 +161,13 @@
         <module-def name="org.jgroups" slot="${infinispan.slot}">
             <maven-resource group="org.jgroups" artifact="jgroups" />
         </module-def>
+
+        <module-def name="org.jgroups.kubernetes" slot="${infinispan.slot}">
+            <maven-resource group="org.jgroups.kubernetes" artifact="kubernetes" />
+            <maven-resource group="org.jgroups.kubernetes" artifact="common" />
+            <maven-resource group="org.jgroups.kubernetes" artifact="dns" />
+            <maven-resource group="net.oauth.core" artifact="oauth" />
+        </module-def>
 
         <module-def name="org.fusesource.leveldbjni-all" slot="${infinispan.slot}">
             <maven-resource group="org.fusesource.leveldbjni" artifact="leveldbjni-all" />

diff --git a/as-modules/embedded/pom.xml b/as-modules/embedded/pom.xml
@@ -158,6 +158,23 @@
          <scope>provided</scope>
       </dependency>
 
+      <dependency>
+         <groupId>org.jgroups.kubernetes</groupId>
+         <artifactId>kubernetes</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.jgroups.kubernetes</groupId>
+         <artifactId>common</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>org.jgroups.kubernetes</groupId>
+         <artifactId>dns</artifactId>
+      </dependency>
+      <dependency>
+         <groupId>net.oauth.core</groupId>
+         <artifactId>oauth</artifactId>
+      </dependency>
+
    </dependencies>
 
    <build>

diff --git a/as-modules/embedded/src/main/resources/org/jgroups/kubernetes/main/module.xml b/as-modules/embedded/src/main/resources/org/jgroups/kubernetes/main/module.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<module xmlns="urn:jboss:module:1.3" name="org.jgroups.kubernetes.kubernetes">
+    <properties>
+        <property name="jboss.api" value="private"/>
+    </properties>
+
+    <resources>
+        <artifact name="${org.jgroups.kubernetes:kubernetes}"/>
+        <artifact name="${org.jgroups.kubernetes:common}"/>
+        <artifact name="${org.jgroups.kubernetes:dns}"/>
+        <artifact name="${net.oauth.core:oauth}"/>
+    </resources>
+
+    <dependencies>
+        <module name="javax.api"/>
+        <module name="io.undertow.core"/>
+        <module name="org.jboss.dmr"/>
+        <module name="org.jgroups"/>
+    </dependencies>
+</module>
diff --git a/bom/pom.xml b/bom/pom.xml
@@ -184,6 +184,11 @@
             <artifactId>infinispan-core</artifactId>
             <version>${project.version}</version>
          </dependency>
+         <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>infinispan-cloud</artifactId>
+            <version>${project.version}</version>
+         </dependency>
          <dependency>
             <groupId>${project.groupId}</groupId>
             <artifactId>infinispan-osgi</artifactId>

diff --git a/cloud/pom.xml b/cloud/pom.xml
@@ -0,0 +1,28 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+   <modelVersion>4.0.0</modelVersion>
+
+   <parent>
+      <groupId>org.infinispan</groupId>
+      <artifactId>infinispan-parent</artifactId>
+      <version>9.0.0-SNAPSHOT</version>
+      <relativePath>../parent/pom.xml</relativePath>
+   </parent>
+
+   <artifactId>infinispan-cloud</artifactId>
+   <name>Infinispan Cloud</name>
+   <description>Infinispan cloud integration module</description>
+
+   <dependencies>
+      <dependency>
+         <groupId>org.infinispan</groupId>
+         <artifactId>infinispan-core</artifactId>
+      </dependency>
+
+      <dependency>
+         <groupId>org.jgroups.kubernetes</groupId>
+         <artifactId>kubernetes</artifactId>
+      </dependency>
+   </dependencies>
+
+</project>
diff --git a/...s/default-configs/default-jgroups-ec2.xml → ...s/default-configs/default-jgroups-ec2.xml b/...s/default-configs/default-jgroups-ec2.xml → ...s/default-configs/default-jgroups-ec2.xml
diff --git a/...efault-configs/default-jgroups-google.xml → ...efault-configs/default-jgroups-google.xml b/...efault-configs/default-jgroups-google.xml → ...efault-configs/default-jgroups-google.xml
diff --git a/cloud/src/main/resources/default-configs/default-jgroups-kubernetes.xml b/cloud/src/main/resources/default-configs/default-jgroups-kubernetes.xml
@@ -0,0 +1,66 @@
+<config xmlns="urn:org:jgroups"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="urn:org:jgroups http://www.jgroups.org/schema/JGroups-3.6.xsd">
+
+   <TCP bind_addr="${jgroups.tcp.address:match-interface:eth.*}"
+        bind_port="${jgroups.tcp.port:7800}"
+        enable_diagnostics="false"
+        thread_naming_pattern="pl"
+        send_buf_size="640k"
+        sock_conn_timeout="300"
+        bundler_type="transfer-queue"
+
+        thread_pool.min_threads="${jgroups.thread_pool.min_threads:2}"
+        thread_pool.max_threads="${jgroups.thread_pool.max_threads:30}"
+        thread_pool.keep_alive_time="60000"
+        thread_pool.queue_enabled="false"
+
+        internal_thread_pool.min_threads="${jgroups.internal_thread_pool.min_threads:5}"
+        internal_thread_pool.max_threads="${jgroups.internal_thread_pool.max_threads:20}"
+        internal_thread_pool.keep_alive_time="60000"
+        internal_thread_pool.queue_enabled="true"
+        internal_thread_pool.queue_max_size="500"
+
+        oob_thread_pool.min_threads="${jgroups.oob_thread_pool.min_threads:20}"
+        oob_thread_pool.max_threads="${jgroups.oob_thread_pool.max_threads:200}"
+        oob_thread_pool.keep_alive_time="60000"
+        oob_thread_pool.queue_enabled="false"
+   />
+   <kubernetes.KUBE_PING
+   />
+   <MERGE3 min_interval="10000" 
+           max_interval="30000" 
+   />
+   <FD_SOCK />
+   <FD_ALL timeout="60000" 
+           interval="15000" 
+           timeout_check_interval="5000" 
+   />
+   <VERIFY_SUSPECT timeout="5000" />
+   <pbcast.NAKACK2 use_mcast_xmit="false"
+                   xmit_interval="1000"
+                   xmit_table_num_rows="50"
+                   xmit_table_msgs_per_row="1024"
+                   xmit_table_max_compaction_time="30000"
+                   max_msg_batch_size="100"
+                   resend_last_seqno="true"
+   />
+   <UNICAST3 xmit_interval="500"
+             xmit_table_num_rows="50"
+             xmit_table_msgs_per_row="1024"
+             xmit_table_max_compaction_time="30000"
+             max_msg_batch_size="100"
+             conn_expiry_timeout="0"
+   />
+   <pbcast.STABLE stability_delay="500"
+                  desired_avg_gossip="5000"
+                  max_bytes="1M"
+   />
+   <pbcast.GMS print_local_addr="false"
+               join_timeout="${jgroups.join_timeout:5000}"
+   />
+   <MFC max_credits="2m" 
+        min_threshold="0.40"
+   />
+   <FRAG2 />
+</config>
diff --git a/core/pom.xml b/core/pom.xml
@@ -30,6 +30,10 @@
          <artifactId>jgroups</artifactId>
       </dependency>
 
+      <dependency>
+         <groupId>org.jgroups.kubernetes</groupId>
+         <artifactId>kubernetes</artifactId>
+      </dependency>
 
       <dependency>
          <groupId>org.jboss.spec.javax.transaction</groupId>

diff --git a/documentation/src/main/asciidoc/upgrading/upgrading.asciidoc b/documentation/src/main/asciidoc/upgrading/upgrading.asciidoc
@@ -8,6 +8,9 @@ The Infinispan community
 This guide walks you through the process of upgrading Infinispan.
 
 == Upgrading from 8.x to 9.0
+=== New Cloud module for library mode
+In Infinispan 8.x, cloud related configuration were added to `infinispan-core` module. Since 9.0 they were moved to `infinispan-cloud` module.
+
 === Entry Retriever is now removed
 The entry retriever feature has been removed.  Please update to use the new Streams feature detailed in the User Guide.
 The `org.infinispan.filter.CacheFilters` class can be used to convert `KeyValueFilter` and `Converter` instances

diff --git a/documentation/src/main/asciidoc/user_guide/cloud_aws.adoc b/documentation/src/main/asciidoc/user_guide/cloud_aws.adoc
@@ -1,5 +1,19 @@
 ==  Running on Cloud Services
 
+In order to turn on Cloud support for Infinispan library mode, one needs to add a new dependency to the classpath:
+
+.Cloud support in library mode
+[source,xml]
+----
+<dependency>
+      <groupId>org.infinispan</groupId>
+      <artifactId>infinispan-cloud</artifactId>
+      <version>${infinispan.version}</version>
+</dependency>
+----
+
+The above dependency adds `infinispan-core` to the classpath as well as some default configurations.
+
 === Amazon Web Services
 Infinispan can be used on the Amazon Web Service (AWS) platform and similar cloud based environment in several ways. As Infinispan uses JGroups as the underlying communication technology, the majority of the configuration work is done JGroups. The default auto discovery won't work on EC2 as multicast is not allowed, but JGroups provides several other discovery protocols so we only have to choose one.
 
@@ -60,5 +74,37 @@ Finally you can configure your JGroups instances to use a shared storage to exch
 ----
 
 ==== JDBC_PING
-A similar approach to S3_PING, but using a JDBC connection to a shared database. On EC2 that is quite easy using Amazon RDS. See the link:$$http://community.jboss.org/wiki/JDBCPING$$[JDBC_PING Wiki page] for details. 
+A similar approach to S3_PING, but using a JDBC connection to a shared database. On EC2 that is quite easy using Amazon RDS. See the link:$$http://community.jboss.org/wiki/JDBCPING$$[JDBC_PING Wiki page] for details.
+
+== Kubernetes and OpenShift
+
+Since OpenShift uses Kubernetes underneath both of them can use the same discovery protocol - link:$$https://github.com/jgroups-extras/jgroups-kubernetes$$[Kube_PING]. The configuration is very straightforward:
+
+.Sample KUBE_PING configuration
+[source,xml]
+----
+
+<config>
+    <TCP bind_addr="${match-interface:eth.*}" />
+    <kubernetes.KUBE_PING />
+...
+...
+</config>
+
+----
+
+The most important thing is to bind JGroups to `eth0` interface, which is link:$$https://docs.docker.com/engine/userguide/networking/dockernetworks/$$[used by Docker containers for network communication].
+
+KUBE_PING protocol is configured by environmental variables (which should be available inside a container). The most important thing is to set `OPENSHIFT_KUBE_PING_NAMESPACE` to proper namespace. It might be either hardcoded or populated via link:$$https://github.com/kubernetes/kubernetes/tree/release-1.0/docs/user-guide/downward-api$$[Kubernetes' Downward API].
+
+Since KUBE_PING uses Kubernetes API for obtaining available Pods, OpenShift requires adding additional privileges. Assuming that `oc project -q` returns current namespace and `default` is the service account name, one needs to run:
+
+.Adding additional OpenShift privileges
+[source,bash]
+----
+
+oc policy add-role-to-user view system:serviceaccount:$(oc project -q):default -n $(oc project -q)
+
+----
 
+After performing all above steps, the clustering should be enabled and all Pods should automatically form a cluster within a single namespace.
diff --git a/documentation/src/main/asciidoc/user_guide/configuration.adoc b/documentation/src/main/asciidoc/user_guide/configuration.adoc
@@ -361,6 +361,7 @@ The configurations available are:
 *  default-jgroups-udp.xml - Uses UDP as a transport, and UDP multicast for discovery.  Usually suitable for larger (over 100 nodes) clusters _or_ if you are using link:http://community.jboss.org/docs/DOC-14853#replicated[replication or invalidation] .  Minimises opening too many sockets. 
 *  default-jgroups-tcp.xml - Uses TCP as a transport and UDP multicast for discovery.  Better for smaller clusters (under 100 nodes) _only if_ you are using link:http://community.jboss.org/docs/DOC-14853#distribution[distribution] , as TCP is more efficient as a point-to-point protocol 
 *  default-jgroups-ec2.xml - Uses TCP as a transport and link:http://community.jboss.org/docs/DOC-15925[S3_PING] for discovery.  Suitable on link:http://aws.amazon.com/ec2/[Amazon EC2] nodes where UDP multicast isn't available. 
+*  default-jgroups-kubernetes.xml - Uses TCP as a transport and link:https://github.com/jgroups-extras/jgroups-kubernetes[KUBE_PING] for discovery.  Suitable on link:http://kubernetes.io/[Kubernetes] and link:https://www.openshift.org/[OpenShift] nodes where UDP multicast is not always available.
 
 ===== Tuning JGroups settings
 The settings above can be further tuned without editing the XML files themselves.
@@ -398,6 +399,13 @@ $ java -cp ... -Djgroups.tcp.port=1234 -Djgroups.tcp.address=10.11.12.13
 |jgroups.s3.bucket|Name of the Amazon S3 bucket to use.  Must be unique and must already exist| |No
 |===============
 
+.default-jgroups-kubernetes.xml
+|===============
+| _System Property_ | _Description_ | _Default_ | _Required?_
+|jgroups.tcp.address|IP address to use for the TCP transport.|eth0|No
+|jgroups.tcp.port|Port to use for TCP socket|7800|No
+|===============
+
 
 ==== Further reading
 JGroups also supports more system property overrides, details of which can be found on this page: link:http://community.jboss.org/docs/12352[SystemProps] 

diff --git a/parent/pom.xml b/parent/pom.xml
@@ -171,6 +171,8 @@
       <version.webdav.servlet>2.0.1</version.webdav.servlet>
       <version.weld>2.3.4.Final</version.weld>
       <version.weld-se>2.3.4.Final</version.weld-se>
+      <version.groups.kubernetes>0.9.0</version.groups.kubernetes>
+      <version.oauth.core>20090531</version.oauth.core>
 
       <version.org.wildfly>10.0.0.Final</version.org.wildfly>
       <version.org.wildfly.core>2.0.10.Final</version.org.wildfly.core>
@@ -983,6 +985,32 @@
              <scope>import</scope>
              <type>pom</type>
           </dependency>
+         <dependency>
+            <groupId>org.jgroups.kubernetes</groupId>
+            <artifactId>kubernetes</artifactId>
+            <version>${version.groups.kubernetes}</version>
+         </dependency>
+         <dependency>
+            <groupId>org.jgroups.kubernetes</groupId>
+            <artifactId>common</artifactId>
+            <version>${version.groups.kubernetes}</version>
+            <exclusions>
+               <exclusion>
+                  <groupId>io.undertow</groupId>
+                  <artifactId>undertow-core</artifactId>
+               </exclusion>
+            </exclusions>
+         </dependency>
+         <dependency>
+            <groupId>org.jgroups.kubernetes</groupId>
+            <artifactId>dns</artifactId>
+            <version>${version.groups.kubernetes}</version>
+         </dependency>
+         <dependency>
+            <groupId>net.oauth.core</groupId>
+            <artifactId>oauth</artifactId>
+            <version>${version.oauth.core}</version>
+         </dependency>
       </dependencies>
    </dependencyManagement>
    <dependencies>

diff --git a/pom.xml b/pom.xml
@@ -22,6 +22,7 @@
       <module>commons</module>
       <module>commons-test</module>
       <module>core</module>
+      <module>cloud</module>
       <module>extended-statistics</module>
       <module>tools</module>
       <module>query</module>