Merge pull request wildfly#16826 from kabir/kafka-add-compression

[WFLY-18009] Add jars needed for compression of Kafka
soul2zimate · Nov 2, 2023 · 86b93af · 86b93af
2 parents 4635273 + ae2b283
commit 86b93af
Show file tree

Hide file tree

Showing 12 changed files with 474 additions and 2 deletions.
diff --git a/boms/common-expansion/pom.xml b/boms/common-expansion/pom.xml
@@ -274,6 +274,19 @@
                 </exclusions>
             </dependency>
 
+
+            <dependency>
+                <groupId>com.github.luben</groupId>
+                <artifactId>zstd-jni</artifactId>
+                <version>${version.com.github.luben.zstd-jni}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>*</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
             <dependency>
                 <groupId>com.fasterxml.jackson.dataformat</groupId>
                 <artifactId>jackson-dataformat-yaml</artifactId>
@@ -1371,12 +1384,35 @@
                 </exclusions>
             </dependency>
 
+            <dependency>
+                <groupId>org.lz4</groupId>
+                <artifactId>lz4-java</artifactId>
+                <version>${version.org.lz4.lz4-java}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>*</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
+
             <dependency>
                 <groupId>org.wildfly.security.mp</groupId>
                 <artifactId>wildfly-elytron-jwt</artifactId>
                 <version>${version.org.wildfly.security.elytron-mp}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.xerial.snappy</groupId>
+                <artifactId>snappy-java</artifactId>
+                <version>${version.org.xerial.snappy.snappy-java}</version>
+                <exclusions>
+                    <exclusion>
+                        <groupId>*</groupId>
+                        <artifactId>*</artifactId>
+                    </exclusion>
+                </exclusions>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 

diff --git a/galleon-pack/galleon-shared/pom.xml b/galleon-pack/galleon-shared/pom.xml
@@ -40,7 +40,19 @@
         </dependency>
 
         <dependency><groupId>com.fasterxml.jackson.jr</groupId><artifactId>jackson-jr-objects</artifactId></dependency>
-
+
+
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency><groupId>com.google.api.grpc</groupId><artifactId>proto-google-common-protos</artifactId></dependency>
         <dependency><groupId>com.google.protobuf</groupId><artifactId>protobuf-java</artifactId></dependency>
         <dependency><groupId>com.google.protobuf</groupId><artifactId>protobuf-java-util</artifactId></dependency>
@@ -636,6 +648,17 @@
         <dependency><groupId>org.jetbrains.kotlin</groupId><artifactId>kotlin-stdlib-jdk8</artifactId></dependency>
         <dependency><groupId>org.jetbrains.kotlin</groupId><artifactId>kotlin-stdlib-jdk7</artifactId></dependency>
 
+        <dependency>
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>*</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>org.wildfly.security.mp</groupId>
             <artifactId>wildfly-elytron-jwt</artifactId>
@@ -647,6 +670,17 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <exclusions>
+                <exclusion>
+                    <artifactId>*</artifactId>
+                    <groupId>*</groupId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
         <dependency>
             <groupId>${full.maven.groupId}</groupId>
             <artifactId>wildfly-microprofile-config-smallrye</artifactId>
@@ -780,7 +814,6 @@
             <groupId>${full.maven.groupId}</groupId>
             <artifactId>wildfly-microprofile-telemetry-cdi-provider</artifactId>
         </dependency>
-
     </dependencies>
 
     <build>

diff --git a/galleon-pack/galleon-shared/src/main/resources/license/licenses.xml b/galleon-pack/galleon-shared/src/main/resources/license/licenses.xml
@@ -28,6 +28,17 @@
                 </license>
             </licenses>
         </dependency>
+        <dependency>
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <licenses>
+                <license>
+                    <name>The BSD License</name>
+                    <url>http://repository.jboss.org/licenses/bsd.txt</url>
+                    <distribution>repo</distribution>
+                </license>
+            </licenses>
+        </dependency>
         <dependency>
             <groupId>com.google.api.grpc</groupId>
             <artifactId>proto-google-common-protos</artifactId>
@@ -961,6 +972,17 @@
                 </license>
             </licenses>
         </dependency>
+        <dependency>
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <licenses>
+                <license>
+                    <name>Apache License 2.0</name>
+                    <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+                    <distribution>repo</distribution>
+                </license>
+            </licenses>
+        </dependency>
         <dependency>
             <groupId>${full.maven.groupId}</groupId>
             <artifactId>wildfly-microprofile-config-smallrye</artifactId>
@@ -1236,5 +1258,16 @@
                 </license>
             </licenses>
         </dependency>
+        <dependency>
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <licenses>
+                <license>
+                    <name>Apache License 2.0</name>
+                    <url>http://www.apache.org/licenses/LICENSE-2.0</url>
+                    <distribution>repo</distribution>
+                </license>
+            </licenses>
+        </dependency>
     </dependencies>
 </licenseSummary>
diff --git a/...d/src/main/resources/modules/system/layers/base/com/github/luben/zstd-jni/main/module.xml b/...d/src/main/resources/modules/system/layers/base/com/github/luben/zstd-jni/main/module.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ JBoss, Home of Professional Open Source.
+  ~ Copyright 2023, Red Hat, Inc., and individual contributors
+  ~ as indicated by the @author tags. See the copyright.txt file in the
+  ~ distribution for a full listing of individual contributors.
+  ~
+  ~ This is free software; you can redistribute it and/or modify it
+  ~ under the terms of the GNU Lesser General Public License as
+  ~ published by the Free Software Foundation; either version 2.1 of
+  ~ the License, or (at your option) any later version.
+  ~
+  ~ This software is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ Lesser General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Lesser General Public
+  ~ License along with this software; if not, write to the Free
+  ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+  ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+  -->
+
+<module xmlns="urn:jboss:module:1.9" name="com.github.luben.zstd-jni">
+
+    <properties>
+        <property name="jboss.api" value="private"/>
+    </properties>
+    <resources>
+        <artifact name="${com.github.luben:zstd-jni}"/>
+    </resources>
+</module>
diff --git a/...red/src/main/resources/modules/system/layers/base/org/apache/kafka/client/main/module.xml b/...red/src/main/resources/modules/system/layers/base/org/apache/kafka/client/main/module.xml
@@ -16,9 +16,12 @@
 
     <dependencies>
         <module name="com.fasterxml.jackson.core.jackson-databind"/>
+        <module name="com.github.luben.zstd-jni"/>
         <module name="java.management"/>
         <module name="java.security.jgss"/>
         <module name="java.security.sasl"/>
+        <module name="org.lz4.lz4-java"/>
         <module name="org.slf4j"/>
+        <module name="org.xerial.snappy.snappy-java"/>
     </dependencies>
 </module>
diff --git a/...eon-shared/src/main/resources/modules/system/layers/base/org/lz4/lz4-java/main/module.xml b/...eon-shared/src/main/resources/modules/system/layers/base/org/lz4/lz4-java/main/module.xml
@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+  ~ JBoss, Home of Professional Open Source.
+  ~ Copyright 2023, Red Hat, Inc., and individual contributors
+  ~ as indicated by the @author tags. See the copyright.txt file in the
+  ~ distribution for a full listing of individual contributors.
+  ~
+  ~ This is free software; you can redistribute it and/or modify it
+  ~ under the terms of the GNU Lesser General Public License as
+  ~ published by the Free Software Foundation; either version 2.1 of
+  ~ the License, or (at your option) any later version.
+  ~
+  ~ This software is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ Lesser General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Lesser General Public
+  ~ License along with this software; if not, write to the Free
+  ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+  ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+  -->
+
+<module xmlns="urn:jboss:module:1.9" name="org.lz4.lz4-java">
+
+    <properties>
+        <property name="jboss.api" value="private"/>
+    </properties>
+    <resources>
+        <artifact name="${org.lz4:lz4-java}"/>
+    </resources>
+</module>
diff --git a/...c/main/resources/modules/system/layers/base/org/xerial/snappy/snappy-java/main/module.xml b/...c/main/resources/modules/system/layers/base/org/xerial/snappy/snappy-java/main/module.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ JBoss, Home of Professional Open Source.
+  ~ Copyright 2023, Red Hat, Inc., and individual contributors
+  ~ as indicated by the @author tags. See the copyright.txt file in the
+  ~ distribution for a full listing of individual contributors.
+  ~
+  ~ This is free software; you can redistribute it and/or modify it
+  ~ under the terms of the GNU Lesser General Public License as
+  ~ published by the Free Software Foundation; either version 2.1 of
+  ~ the License, or (at your option) any later version.
+  ~
+  ~ This software is distributed in the hope that it will be useful,
+  ~ but WITHOUT ANY WARRANTY; without even the implied warranty of
+  ~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+  ~ Lesser General Public License for more details.
+  ~
+  ~ You should have received a copy of the GNU Lesser General Public
+  ~ License along with this software; if not, write to the Free
+  ~ Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
+  ~ 02110-1301 USA, or see the FSF site: http://www.fsf.org.
+  -->
+
+<module xmlns="urn:jboss:module:1.9" name="org.xerial.snappy.snappy-java">
+
+    <properties>
+        <property name="jboss.api" value="private"/>
+    </properties>
+    <resources>
+        <artifact name="${org.xerial.snappy:snappy-java}"/>
+    </resources>
+</module>
diff --git a/pom.xml b/pom.xml
@@ -372,6 +372,7 @@
         <version.com.github.fge.jackson-coreutils>1.8</version.com.github.fge.jackson-coreutils>
         <version.com.github.fge.json-patch>1.9</version.com.github.fge.json-patch>
         <version.com.github.fge.msg-simple>1.1</version.com.github.fge.msg-simple>
+        <version.com.github.luben.zstd-jni>1.5.2-1</version.com.github.luben.zstd-jni>
         <version.com.google.api.grpc>2.0.1</version.com.google.api.grpc>
         <version.com.google.code.gson>2.8.9</version.com.google.code.gson>
         <version.com.google.guava>32.1.2-jre</version.com.google.guava>
@@ -535,6 +536,7 @@
         <version.org.jvnet.staxex>2.1.0</version.org.jvnet.staxex>
         <version.org.keycloak.keycloak-saml-wildfly-subsystem>18.0.2</version.org.keycloak.keycloak-saml-wildfly-subsystem>
         <version.org.kohsuke.metainf-services>1.11</version.org.kohsuke.metainf-services>
+        <version.org.lz4.lz4-java>1.8.0</version.org.lz4.lz4-java>
         <version.org.opensaml.opensaml>4.2.0</version.org.opensaml.opensaml>
         <version.org.ow2.asm>9.5</version.org.ow2.asm>
         <version.org.reactivestreams>1.0.4</version.org.reactivestreams>
@@ -546,6 +548,7 @@
         <preview.version.org.wildfly.wildfly-ee-9-deployment-transformer>1.0.0.Final</preview.version.org.wildfly.wildfly-ee-9-deployment-transformer>
         <version.software.amazon.awssdk>2.20.126</version.software.amazon.awssdk>
         <version.software.amazon.eventstream>1.0.1</version.software.amazon.eventstream>
+        <version.org.xerial.snappy.snappy-java>1.1.8.4</version.org.xerial.snappy.snappy-java>
         <version.sun.jaxb>${version.org.glassfish.jaxb}</version.sun.jaxb>
         <version.wsdl4j>1.6.3</version.wsdl4j>
         <version.xml-resolver>1.2</version.xml-resolver>

diff --git a/testsuite/integration/microprofile/pom.xml b/testsuite/integration/microprofile/pom.xml
@@ -226,6 +226,24 @@
             <version>${version.org.apache.kafka}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <!-- Needed if people use compression.type=zstd in their Kafka config -->
+            <groupId>com.github.luben</groupId>
+            <artifactId>zstd-jni</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <!-- Needed if people use compression.type=lz4 in their Kafka config -->
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <!-- Needed if people use compression.type=snappy in their Kafka config -->
+            <groupId>org.xerial.snappy</groupId>
+            <artifactId>snappy-java</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.wildfly.core</groupId>
             <artifactId>wildfly-cli</artifactId>