Skip to content

Commit 73c84d4

Browse files
ulysses-youulysses-you
committed
[KYUUBI #1868] Upgrate netty from 4.1.68 to 4.1.73
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> close #1868 ### _How was this patch tested?_ Pass CI Closes #1869 from ulysses-you/netty. Closes #1868 c197d0e [ulysses-you] Upgrate netty from 4.1.68 to 4.1.73 Authored-by: ulysses-you <ulyssesyou18@gmail.com> Signed-off-by: ulysses-you <ulyssesyou@apache.org>
1 parent c8c1d74 commit 73c84d4

File tree

8 files changed

+111
-31
lines changed

8 files changed

+111
-31
lines changed

‎dev/dependencyList

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,21 @@ metrics-core/4.2.8//metrics-core-4.2.8.jar
7979
metrics-jmx/4.2.8//metrics-jmx-4.2.8.jar
8080
metrics-json/4.2.8//metrics-json-4.2.8.jar
8181
metrics-jvm/4.2.8//metrics-jvm-4.2.8.jar
82-
netty-all/4.1.68.Final//netty-all-4.1.68.Final.jar
82+
netty-all/4.1.73.Final//netty-all-4.1.73.Final.jar
83+
netty-buffer/4.1.73.Final//netty-buffer-4.1.73.Final.jar
84+
netty-codec/4.1.73.Final//netty-codec-4.1.73.Final.jar
85+
netty-common/4.1.73.Final//netty-common-4.1.73.Final.jar
86+
netty-handler/4.1.73.Final//netty-handler-4.1.73.Final.jar
87+
netty-resolver/4.1.73.Final//netty-resolver-4.1.73.Final.jar
88+
netty-tcnative-classes/2.0.46.Final//netty-tcnative-classes-2.0.46.Final.jar
89+
netty-transport-classes-epoll/4.1.73.Final//netty-transport-classes-epoll-4.1.73.Final.jar
90+
netty-transport-classes-kqueue/4.1.73.Final//netty-transport-classes-kqueue-4.1.73.Final.jar
91+
netty-transport-native-epoll/4.1.73.Final/linux-aarch_64/netty-transport-native-epoll-4.1.73.Final-linux-aarch_64.jar
92+
netty-transport-native-epoll/4.1.73.Final/linux-x86_64/netty-transport-native-epoll-4.1.73.Final-linux-x86_64.jar
93+
netty-transport-native-kqueue/4.1.73.Final/osx-aarch_64/netty-transport-native-kqueue-4.1.73.Final-osx-aarch_64.jar
94+
netty-transport-native-kqueue/4.1.73.Final/osx-x86_64/netty-transport-native-kqueue-4.1.73.Final-osx-x86_64.jar
95+
netty-transport-native-unix-common/4.1.73.Final//netty-transport-native-unix-common-4.1.73.Final.jar
96+
netty-transport/4.1.73.Final//netty-transport-4.1.73.Final.jar
8397
osgi-resource-locator/1.0.3//osgi-resource-locator-1.0.3.jar
8498
paranamer/2.8//paranamer-2.8.jar
8599
scala-library/2.12.15//scala-library-2.12.15.jar

‎externals/kyuubi-flink-sql-engine/pom.xml

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,6 @@
165165
<include>com.google.guava:failureaccess</include>
166166
<include>com.google.guava:guava</include>
167167
<include>commons-codec:commons-codec</include>
168-
<include>io.netty:netty-all</include>
169168
<include>org.apache.commons:commons-lang3</include>
170169
<include>org.apache.curator:curator-client</include>
171170
<include>org.apache.curator:curator-framework</include>
@@ -205,13 +204,6 @@
205204
<pattern>org.apache.commons.lang</pattern>
206205
<shadedPattern>${kyuubi.shade.packageName}.org.apache.commons.lang</shadedPattern>
207206
</relocation>
208-
<relocation>
209-
<pattern>io.netty</pattern>
210-
<shadedPattern>${kyuubi.shade.packageName}.io.netty</shadedPattern>
211-
<includes>
212-
<include>io.netty.**</include>
213-
</includes>
214-
</relocation>
215207
<relocation>
216208
<pattern>org.apache.hive.service.rpc.thrift</pattern>
217209
<shadedPattern>${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift</shadedPattern>

‎kyuubi-common/pom.xml

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -113,11 +113,6 @@
113113
<artifactId>jackson-databind</artifactId>
114114
</dependency>
115115

116-
<dependency>
117-
<groupId>io.netty</groupId>
118-
<artifactId>netty-all</artifactId>
119-
</dependency>
120-
121116
<dependency>
122117
<groupId>org.apache.hadoop</groupId>
123118
<artifactId>hadoop-minikdc</artifactId>

‎kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ import scala.collection.JavaConverters._
2727
import org.apache.kyuubi.{Logging, Utils}
2828
import org.apache.kyuubi.engine.{EngineType, ShareLevel}
2929
import org.apache.kyuubi.service.authentication.{AuthTypes, SaslQOP}
30-
import org.apache.kyuubi.util.NettyUtils.MAX_NETTY_THREADS
3130

3231
case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging {
3332
import KyuubiConf._
@@ -516,6 +515,16 @@ object KyuubiConf {
516515
.checkValue(p => p == 0 || (p > 1024 && p < 65535), "Invalid Port number")
517516
.createWithDefault(3309)
518517

518+
/**
519+
* Specifies an upper bound on the number of Netty threads that Kyuubi requires by default.
520+
* In practice, only 2-4 cores should be required to transfer roughly 10 Gb/s, and each core
521+
* that we use will have an initial overhead of roughly 32 MB of off-heap memory, which comes
522+
* at a premium.
523+
*
524+
* Thus, this value should still retain maximum throughput and reduce wasted off-heap memory
525+
* allocation.
526+
*/
527+
val MAX_NETTY_THREADS: Int = 8
519528
val FRONTEND_MYSQL_NETTY_WORKER_THREADS: OptionalConfigEntry[Int] =
520529
buildConf("frontend.mysql.netty.worker.threads")
521530
.doc("Number of thread in the netty worker event loop of MySQL frontend service. " +

‎kyuubi-server/pom.xml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,11 @@
130130
<artifactId>commons-lang</artifactId>
131131
</dependency>
132132

133+
<dependency>
134+
<groupId>io.netty</groupId>
135+
<artifactId>netty-all</artifactId>
136+
</dependency>
137+
133138
<dependency>
134139
<groupId>io.swagger.core.v3</groupId>
135140
<artifactId>swagger-jaxrs2</artifactId>

‎kyuubi-server/src/main/scala/org/apache/kyuubi/server/KyuubiMySQLFrontendService.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,10 +30,10 @@ import org.apache.kyuubi._
3030
import org.apache.kyuubi.config.KyuubiConf
3131
import org.apache.kyuubi.config.KyuubiConf._
3232
import org.apache.kyuubi.server.mysql._
33+
import org.apache.kyuubi.server.mysql.NettyUtils._
3334
import org.apache.kyuubi.server.mysql.authentication.MySQLAuthHandler
3435
import org.apache.kyuubi.service.{AbstractFrontendService, Serverable, Service}
3536
import org.apache.kyuubi.util.NamedThreadFactory
36-
import org.apache.kyuubi.util.NettyUtils._
3737

3838
/**
3939
* A frontend service implement MySQL protocol.

‎kyuubi-common/src/main/scala/org/apache/kyuubi/util/NettyUtils.scala renamed to ‎kyuubi-server/src/main/scala/org/apache/kyuubi/server/mysql/NettyUtils.scala

Lines changed: 5 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.kyuubi.util
18+
package org.apache.kyuubi.server.mysql
1919

2020
import java.util.concurrent.ThreadFactory
2121

@@ -27,18 +27,9 @@ import io.netty.channel.nio.NioEventLoopGroup
2727
import io.netty.channel.socket.nio.{NioServerSocketChannel, NioSocketChannel}
2828
import io.netty.util.concurrent.DefaultThreadFactory
2929

30-
object NettyUtils {
30+
import org.apache.kyuubi.config.KyuubiConf
3131

32-
/**
33-
* Specifies an upper bound on the number of Netty threads that Kyuubi requires by default.
34-
* In practice, only 2-4 cores should be required to transfer roughly 10 Gb/s, and each core
35-
* that we use will have an initial overhead of roughly 32 MB of off-heap memory, which comes
36-
* at a premium.
37-
*
38-
* Thus, this value should still retain maximum throughput and reduce wasted off-heap memory
39-
* allocation.
40-
*/
41-
val MAX_NETTY_THREADS: Int = 8
32+
object NettyUtils {
4233

4334
val EPOLL_MODE: Boolean = Epoll.isAvailable
4435

@@ -65,7 +56,7 @@ object NettyUtils {
6556
* we will use Runtime get an approximate number of available cores.
6657
*/
6758
def defaultNumThreads(numUsableCores: Option[Int]): Int = numUsableCores match {
68-
case Some(num) => min(num, MAX_NETTY_THREADS)
69-
case None => min(sys.runtime.availableProcessors, MAX_NETTY_THREADS)
59+
case Some(num) => min(num, KyuubiConf.MAX_NETTY_THREADS)
60+
case None => min(sys.runtime.availableProcessors, KyuubiConf.MAX_NETTY_THREADS)
7061
}
7162
}

‎pom.xml

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@
124124
<kubernetes-client.version>5.12.1</kubernetes-client.version>
125125
<ldapsdk.version>5.1.4</ldapsdk.version>
126126
<mysql.jdbc.version>8.0.27</mysql.jdbc.version>
127-
<netty.version>4.1.68.Final</netty.version>
127+
<netty.version>4.1.73.Final</netty.version>
128128
<parquet.version>1.10.1</parquet.version>
129129
<prometheus.version>0.14.1</prometheus.version>
130130
<scalacheck.version>3.2.9.0</scalacheck.version>
@@ -745,6 +745,80 @@
745745
<groupId>io.netty</groupId>
746746
<artifactId>netty-all</artifactId>
747747
<version>${netty.version}</version>
748+
<exclusions>
749+
<exclusion>
750+
<groupId>io.netty</groupId>
751+
<artifactId>netty-codec-dns</artifactId>
752+
</exclusion>
753+
<exclusion>
754+
<groupId>io.netty</groupId>
755+
<artifactId>netty-codec-haproxy</artifactId>
756+
</exclusion>
757+
<exclusion>
758+
<groupId>io.netty</groupId>
759+
<artifactId>netty-codec-http</artifactId>
760+
</exclusion>
761+
<exclusion>
762+
<groupId>io.netty</groupId>
763+
<artifactId>netty-codec-http2</artifactId>
764+
</exclusion>
765+
<exclusion>
766+
<groupId>io.netty</groupId>
767+
<artifactId>netty-codec-memcache</artifactId>
768+
</exclusion>
769+
<exclusion>
770+
<groupId>io.netty</groupId>
771+
<artifactId>netty-codec-mqtt</artifactId>
772+
</exclusion>
773+
<exclusion>
774+
<groupId>io.netty</groupId>
775+
<artifactId>netty-codec-redis</artifactId>
776+
</exclusion>
777+
<exclusion>
778+
<groupId>io.netty</groupId>
779+
<artifactId>netty-codec-smtp</artifactId>
780+
</exclusion>
781+
<exclusion>
782+
<groupId>io.netty</groupId>
783+
<artifactId>netty-codec-socks</artifactId>
784+
</exclusion>
785+
<exclusion>
786+
<groupId>io.netty</groupId>
787+
<artifactId>netty-codec-stomp</artifactId>
788+
</exclusion>
789+
<exclusion>
790+
<groupId>io.netty</groupId>
791+
<artifactId>netty-codec-xml</artifactId>
792+
</exclusion>
793+
<exclusion>
794+
<groupId>io.netty</groupId>
795+
<artifactId>netty-handler-proxy</artifactId>
796+
</exclusion>
797+
<exclusion>
798+
<groupId>io.netty</groupId>
799+
<artifactId>netty-resolver-dns</artifactId>
800+
</exclusion>
801+
<exclusion>
802+
<groupId>io.netty</groupId>
803+
<artifactId>netty-resolver-dns-classes-macos</artifactId>
804+
</exclusion>
805+
<exclusion>
806+
<groupId>io.netty</groupId>
807+
<artifactId>netty-resolver-dns-native-macos</artifactId>
808+
</exclusion>
809+
<exclusion>
810+
<groupId>io.netty</groupId>
811+
<artifactId>netty-transport-rxtx</artifactId>
812+
</exclusion>
813+
<exclusion>
814+
<groupId>io.netty</groupId>
815+
<artifactId>netty-transport-sctp</artifactId>
816+
</exclusion>
817+
<exclusion>
818+
<groupId>io.netty</groupId>
819+
<artifactId>netty-transport-udt</artifactId>
820+
</exclusion>
821+
</exclusions>
748822
</dependency>
749823

750824
<dependency>

0 commit comments

Comments
 (0)