[WFLY-18009] Add jars needed for compression of Kafka

wildfly · May 15, 2023 · 38fe2c5 · 38fe2c5
1 parent 57dec12
commit 38fe2c5
Show file tree

Hide file tree

Showing 12 changed files with 479 additions and 1 deletion.
diff --git a/boms/standard-expansion/pom.xml b/boms/standard-expansion/pom.xml
@@ -204,7 +204,19 @@
                     </exclusion>
                 </exclusions>
             </dependency>
-
+
+            <dependency>
+                <groupId>com.github.luben</groupId>
+                <artifactId>zstd-jni</artifactId>
+                <version>${version.com.github.luben.zstd-jni}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>*</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
             <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protobuf-java</artifactId>
@@ -759,12 +771,37 @@
                 </exclusions>
             </dependency>
 
+            <dependency>
+                <groupId>org.lz4</groupId>
+                <artifactId>lz4-java</artifactId>
+                <version>${version.org.lz4.lz4-java}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>*</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
             <dependency>
                 <groupId>org.wildfly.security.mp</groupId>
                 <artifactId>wildfly-elytron-jwt</artifactId>
                 <version>${version.org.wildfly.security.elytron-mp}</version>
             </dependency>
+
+            <dependency>
+                <groupId>org.xerial.snappy</groupId>
+                <artifactId>snappy-java</artifactId>
+                <version>${version.org.xerial.snappy.snappy-java}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>*</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
+
     </dependencyManagement>
 
 </project>
diff --git a/microprofile/galleon-common/pom.xml b/microprofile/galleon-common/pom.xml
@@ -185,6 +185,17 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>io.netty</groupId>
             <artifactId>netty-resolver-dns</artifactId>
@@ -704,6 +715,18 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <version>${version.org.lz4.lz4-java}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>${full.maven.groupId}</groupId>
             <artifactId>wildfly-microprofile-config-smallrye</artifactId>
@@ -801,6 +824,18 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <version>${version.org.xerial.snappy.snappy-java}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>org.eclipse.microprofile.rest.client</groupId>
             <artifactId>microprofile-rest-client-api</artifactId>

diff --git a/...oprofile/galleon-common/src/main/resources/license/microprofile-feature-pack-licenses.xml b/...oprofile/galleon-common/src/main/resources/license/microprofile-feature-pack-licenses.xml
@@ -12,6 +12,17 @@
         </license>
       </licenses>
     </dependency>
+    <dependency>
+      <groupId>com.github.luben</groupId>
+      <artifactId>zstd-jni</artifactId>
+      <licenses>
+        <license>
+          <name>The BSD License</name>
+          <url>http://repository.jboss.org/licenses/bsd.txt</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </dependency>
     <dependency>
       <groupId>io.netty</groupId>
       <artifactId>netty-codec-dns</artifactId>
@@ -758,6 +769,17 @@
         </license>
       </licenses>
     </dependency>
+    <dependency>
+      <groupId>org.lz4</groupId>
+      <artifactId>lz4-java</artifactId>
+      <licenses>
+        <license>
+          <name>Apache License 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </dependency>
     <dependency>
       <groupId>org.reactivestreams</groupId>
       <artifactId>reactive-streams</artifactId>
@@ -769,6 +791,17 @@
         </license>
       </licenses>
     </dependency>
+    <dependency>
+      <groupId>org.xerial.snappy</groupId>
+      <artifactId>snappy-java</artifactId>
+      <licenses>
+        <license>
+          <name>Apache License 2.0</name>
+          <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+          <distribution>repo</distribution>
+        </license>
+      </licenses>
+    </dependency>
     <dependency>
       <groupId>org.wildfly.security.mp</groupId>
       <artifactId>wildfly-elytron-jwt</artifactId>

diff --git a/...n/src/main/resources/modules/system/layers/base/com/github/luben/zstd-jni/main/module.xml b/...n/src/main/resources/modules/system/layers/base/com/github/luben/zstd-jni/main/module.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ JBoss, Home of Professional Open Source.
+  ~ Copyright 2023, Red Hat, Inc., and individual contributors
+  ~ as indicated by the @author tags. See the copyright.txt file in the
+  ~ distribution for a full listing of individual contributors.
+  ~
+  ~ This is free software; you can redistribute it and/or modify it
+  ~ under the terms of the GNU Lesser General Public License as
+  ~ published by the Free Software Foundation; either version 2.1 of
+  ~ the License, or (at your option) any later version.
+  ~
+  ~ This software is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ Lesser General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Lesser General Public
+  ~ License along with this software; if not, write to the Free
+  ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+  ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+  -->
+
+<module xmlns="urn:jboss:module:1.9" name="com.github.luben.zstd-jni">
+
+  <properties>
+    <property name="jboss.api" value="private"/>
+  </properties>
+  <resources>
+    <artifact name="${com.github.luben:zstd-jni}"/>
+  </resources>
+</module>
diff --git a/...mon/src/main/resources/modules/system/layers/base/org/apache/kafka/client/main/module.xml b/...mon/src/main/resources/modules/system/layers/base/org/apache/kafka/client/main/module.xml
@@ -33,9 +33,12 @@
 
     <dependencies>
         <module name="com.fasterxml.jackson.core.jackson-databind"/>
+        <module name="com.github.luben.zstd-jni"/>
         <module name="java.management"/>
         <module name="java.security.jgss"/>
         <module name="java.security.sasl"/>
+        <module name="org.lz4.lz4-java"/>
         <module name="org.slf4j"/>
+        <module name="org.xerial.snappy.snappy-java"/>
     </dependencies>
 </module>
diff --git a/...eon-common/src/main/resources/modules/system/layers/base/org/lz4/lz4-java/main/module.xml b/...eon-common/src/main/resources/modules/system/layers/base/org/lz4/lz4-java/main/module.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ JBoss, Home of Professional Open Source.
+  ~ Copyright 2023, Red Hat, Inc., and individual contributors
+  ~ as indicated by the @author tags. See the copyright.txt file in the
+  ~ distribution for a full listing of individual contributors.
+  ~
+  ~ This is free software; you can redistribute it and/or modify it
+  ~ under the terms of the GNU Lesser General Public License as
+  ~ published by the Free Software Foundation; either version 2.1 of
+  ~ the License, or (at your option) any later version.
+  ~
+  ~ This software is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ Lesser General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Lesser General Public
+  ~ License along with this software; if not, write to the Free
+  ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+  ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+  -->
+
+<module xmlns="urn:jboss:module:1.9" name="org.lz4.lz4-java">
+
+  <properties>
+    <property name="jboss.api" value="private"/>
+  </properties>
+  <resources>
+    <artifact name="${org.lz4:lz4-java}"/>
+  </resources>
+</module>
diff --git a/...c/main/resources/modules/system/layers/base/org/xerial/snappy/snappy-java/main/module.xml b/...c/main/resources/modules/system/layers/base/org/xerial/snappy/snappy-java/main/module.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ JBoss, Home of Professional Open Source.
+  ~ Copyright 2023, Red Hat, Inc., and individual contributors
+  ~ as indicated by the @author tags. See the copyright.txt file in the
+  ~ distribution for a full listing of individual contributors.
+  ~
+  ~ This is free software; you can redistribute it and/or modify it
+  ~ under the terms of the GNU Lesser General Public License as
+  ~ published by the Free Software Foundation; either version 2.1 of
+  ~ the License, or (at your option) any later version.
+  ~
+  ~ This software is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ Lesser General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Lesser General Public
+  ~ License along with this software; if not, write to the Free
+  ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+  ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+  -->
+
+<module xmlns="urn:jboss:module:1.9" name="org.xerial.snappy.snappy-java">
+
+  <properties>
+    <property name="jboss.api" value="private"/>
+  </properties>
+  <resources>
+    <artifact name="${org.xerial.snappy:snappy-java}"/>
+  </resources>
+</module>
diff --git a/pom.xml b/pom.xml
@@ -329,6 +329,7 @@
         <version.com.github.fge.jackson-coreutils>1.8</version.com.github.fge.jackson-coreutils>
         <version.com.github.fge.json-patch>1.9</version.com.github.fge.json-patch>
         <version.com.github.fge.msg-simple>1.1</version.com.github.fge.msg-simple>
+        <version.com.github.luben.zstd-jni>1.5.2-1</version.com.github.luben.zstd-jni>
         <version.com.github.tomakehurst.wiremock>2.27.2</version.com.github.tomakehurst.wiremock>
         <version.com.google.android.annotations>4.1.1.4</version.com.google.android.annotations>
         <version.com.google.api.grpc>2.0.1</version.com.google.api.grpc>
@@ -532,6 +533,7 @@
         <version.org.keycloak.keycloak-saml-wildfly-subsystem>18.0.2</version.org.keycloak.keycloak-saml-wildfly-subsystem>
         <version.org.kohsuke.metainf-services>1.8</version.org.kohsuke.metainf-services>
         <version.org.latencyutils>2.0.3</version.org.latencyutils>
+        <version.org.lz4.lz4-java>1.8.0</version.org.lz4.lz4-java>
         <version.org.opensaml.opensaml>4.2.0</version.org.opensaml.opensaml>
         <version.org.ow2.asm>9.4</version.org.ow2.asm>
         <version.org.reactivestreams>1.0.4</version.org.reactivestreams>
@@ -547,6 +549,7 @@
         <version.org.wildfly.security.elytron-mp>2.0.0.Final</version.org.wildfly.security.elytron-mp>
         <version.org.wildfly.transaction.client>3.0.0.Final</version.org.wildfly.transaction.client>
         <version.org.wildfly.wildfly-ee-9-deployment-transformer>1.0.0.Final</version.org.wildfly.wildfly-ee-9-deployment-transformer>
+        <version.org.xerial.snappy.snappy-java>1.1.8.4</version.org.xerial.snappy.snappy-java>
         <version.sun.jaxb>${version.org.glassfish.jaxb}</version.sun.jaxb>
         <version.wsdl4j>1.6.3</version.wsdl4j>
         <version.xml-resolver>1.2</version.xml-resolver>

diff --git a/testsuite/integration/microprofile/pom.xml b/testsuite/integration/microprofile/pom.xml
@@ -239,6 +239,24 @@
             <version>${version.org.apache.kafka}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <!-- Needed if people use compression.type=zstd in their Kafka config -->
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <!-- Needed if people use compression.type=lz4 in their Kafka config -->
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <!-- Needed if people use compression.type=snappy in their Kafka config -->
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.wildfly.core</groupId>
             <artifactId>wildfly-cli</artifactId>