[KYUUBI #2103] [1.5] Revert "[KYUUBI #1948] Upgrade thrift version to…

… 0.16.0"

### _Why are the changes needed?_

This PR reverts "[KYUUBI #1948] Upgrade thrift version to 0.16.0" in branch-1.5 and some subsequent commits due to issues caused by breaking changes in Thrift, e.g. #2101, #2093

We need more time to evaluate it and add more test cases to ensure stability before the next release.

### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request

Closes #2103 from pan3793/revert.

Closes #2103

Closes #1948

575dfe8 [Cheng Pan] properties order
4c94481 [Cheng Pan] fix deps
bfbdeb8 [Cheng Pan] Add log4j deps for kyuubi-hive-jdbc test
bad4e90 [Cheng Pan] surefire
f2ae8b9 [Cheng Pan] Revert "[KYUUBI #1948] Upgrade thrift version to 0.16.0"
4501f74 [Cheng Pan] Revert "[KYUUBI #1948][FOLLOWUP] Remove TFramedTransport in extension-spark-3-1 and extension-spark-3-2"
cc04df4 [Cheng Pan] Revert "[KYUUBI #1996] Clean up log4j2 conf and enhance surefire plugin conf"
52a20e1 [Cheng Pan] Revert "[KYUUBI #1948][FOLLOWUP] Relocate fb303 classes"

Authored-by: Cheng Pan <chengpan@apache.org>
Signed-off-by: Cheng Pan <chengpan@apache.org>

conf/log4j2.properties.template

@@ -25,8 +25,17 @@ appender.console.name = STDOUT
 appender.console.target = SYSTEM_OUT
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
-appender.console.filter.a.type = ThresholdFilter
-appender.console.filter.a.level = info
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = info
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
 
 # Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the
 # log level for this class is used to overwrite the root logger's log level.

dev/dependencyList

@@ -68,7 +68,7 @@ jetty-util-ajax/9.4.41.v20210516//jetty-util-ajax-9.4.41.v20210516.jar
 jetty-util/9.4.41.v20210516//jetty-util-9.4.41.v20210516.jar
 jline/0.9.94//jline-0.9.94.jar
 libfb303/0.9.3//libfb303-0.9.3.jar
-libthrift/0.16.0//libthrift-0.16.0.jar
+libthrift/0.9.3//libthrift-0.9.3.jar
 log4j-1.2-api/2.17.2//log4j-1.2-api-2.17.2.jar
 log4j-api/2.17.2//log4j-api-2.17.2.jar
 log4j-core/2.17.2//log4j-core-2.17.2.jar

dev/kyuubi-extension-spark-3-1/src/test/resources/log4j2-test.properties

@@ -26,15 +26,32 @@ appender.console.name = STDOUT
 appender.console.target = SYSTEM_OUT
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
-appender.console.filter.a.type = ThresholdFilter
-appender.console.filter.a.level = fatal
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = fatal
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
 
 # File Appender
 appender.file.type = File
 appender.file.name = File
 appender.file.fileName = target/unit-tests.log
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+appender.file.filter.1.type = Filters
+
+appender.file.filter.1.a.type = RegexFilter
+appender.file.filter.1.a.regex = .*Thrift error occurred during processing of message.*
+appender.file.filter.1.a.onMatch = deny
+appender.file.filter.1.a.onMismatch = neutral
+
 # Set the logger level of File Appender to DEBUG
-appender.file.filter.a.type = ThresholdFilter
-appender.file.filter.a.level = debug
+appender.file.filter.1.b.type = ThresholdFilter
+appender.file.filter.1.b.level = debug

dev/kyuubi-extension-spark-3-2/src/test/resources/log4j2-test.properties

@@ -26,15 +26,32 @@ appender.console.name = STDOUT
 appender.console.target = SYSTEM_OUT
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
-appender.console.filter.a.type = ThresholdFilter
-appender.console.filter.a.level = fatal
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = fatal
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
 
 # File Appender
 appender.file.type = File
 appender.file.name = File
 appender.file.fileName = target/unit-tests.log
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+appender.file.filter.1.type = Filters
+
+appender.file.filter.1.a.type = RegexFilter
+appender.file.filter.1.a.regex = .*Thrift error occurred during processing of message.*
+appender.file.filter.1.a.onMatch = deny
+appender.file.filter.1.a.onMismatch = neutral
+
 # Set the logger level of File Appender to DEBUG
-appender.file.filter.a.type = ThresholdFilter
-appender.file.filter.a.level = debug
+appender.file.filter.1.b.type = ThresholdFilter
+appender.file.filter.1.b.level = debug

dev/kyuubi-extension-spark-common/src/test/resources/log4j2-test.properties

@@ -26,15 +26,32 @@ appender.console.name = STDOUT
 appender.console.target = SYSTEM_OUT
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
-appender.console.filter.a.type = ThresholdFilter
-appender.console.filter.a.level = fatal
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = fatal
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
 
 # File Appender
 appender.file.type = File
 appender.file.name = File
 appender.file.fileName = target/unit-tests.log
 appender.file.layout.type = PatternLayout
 appender.file.layout.pattern = %d{HH:mm:ss.SSS} %t %p %c{1}: %m%n
+
+appender.file.filter.1.type = Filters
+
+appender.file.filter.1.a.type = RegexFilter
+appender.file.filter.1.a.regex = .*Thrift error occurred during processing of message.*
+appender.file.filter.1.a.onMatch = deny
+appender.file.filter.1.a.onMismatch = neutral
+
 # Set the logger level of File Appender to DEBUG
-appender.file.filter.a.type = ThresholdFilter
-appender.file.filter.a.level = debug
+appender.file.filter.1.b.type = ThresholdFilter
+appender.file.filter.1.b.level = debug

docs/deployment/settings.md

@@ -211,9 +211,11 @@ Key | Default | Meaning | Type | Since
 
 Key | Default | Meaning | Type | Since
 --- | --- | --- | --- | ---
+<code>kyuubi.frontend.backoff.slot.length</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT0.1S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) Time to back off during login to the thrift frontend service.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>
 <code>kyuubi.frontend.bind.host</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>&lt;undefined&gt;</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) Hostname or IP of the machine on which to run the thrift frontend service via binary protocol.</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.0.0</div>
 <code>kyuubi.frontend.bind.port</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>10009</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) Port of the machine on which to run the thrift frontend service via binary protocol.</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.0.0</div>
 <code>kyuubi.frontend.connection.url.use.hostname</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>false</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>When true, frontend services prefer hostname, otherwise, ip address</div>|<div style='width: 30pt'>boolean</div>|<div style='width: 20pt'>1.5.0</div>
+<code>kyuubi.frontend.login.timeout</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT20S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) Timeout for Thrift clients during login to the thrift frontend service.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.0.0</div>
 <code>kyuubi.frontend.max.message.size</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>104857600</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) Maximum message size in bytes a Kyuubi server will accept.</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.0.0</div>
 <code>kyuubi.frontend.max.worker.threads</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>999</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) Maximum number of threads in the of frontend worker thread pool for the thrift frontend service</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.0.0</div>
 <code>kyuubi.frontend.min.worker.threads</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>9</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>(deprecated) Minimum number of threads in the of frontend worker thread pool for the thrift frontend service</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.0.0</div>
@@ -226,8 +228,10 @@ Key | Default | Meaning | Type | Since
 <code>kyuubi.frontend.protocols</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>THRIFT_BINARY</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>A comma separated list for all frontend protocols <ul> <li>THRIFT_BINARY - HiveServer2 compatible thrift binary protocol.</li> <li>REST - Kyuubi defined REST API(experimental).</li>  <li>MYSQL - MySQL compatible text protocol(experimental).</li> </ul></div>|<div style='width: 30pt'>seq</div>|<div style='width: 20pt'>1.4.0</div>
 <code>kyuubi.frontend.rest.bind.host</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>&lt;undefined&gt;</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Hostname or IP of the machine on which to run the REST frontend service.</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.4.0</div>
 <code>kyuubi.frontend.rest.bind.port</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>10099</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Port of the machine on which to run the REST frontend service.</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
+<code>kyuubi.frontend.thrift.backoff.slot.length</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT0.1S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Time to back off during login to the thrift frontend service.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.4.0</div>
 <code>kyuubi.frontend.thrift.binary.bind.host</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>&lt;undefined&gt;</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Hostname or IP of the machine on which to run the thrift frontend service via binary protocol.</div>|<div style='width: 30pt'>string</div>|<div style='width: 20pt'>1.4.0</div>
 <code>kyuubi.frontend.thrift.binary.bind.port</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>10009</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Port of the machine on which to run the thrift frontend service via binary protocol.</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
+<code>kyuubi.frontend.thrift.login.timeout</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>PT20S</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Timeout for Thrift clients during login to the thrift frontend service.</div>|<div style='width: 30pt'>duration</div>|<div style='width: 20pt'>1.4.0</div>
 <code>kyuubi.frontend.thrift.max.message.size</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>104857600</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Maximum message size in bytes a Kyuubi server will accept.</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
 <code>kyuubi.frontend.thrift.max.worker.threads</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>999</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Maximum number of threads in the of frontend worker thread pool for the thrift frontend service</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
 <code>kyuubi.frontend.thrift.min.worker.threads</code>|<div style='width: 65pt;word-wrap: break-word;white-space: normal'>9</div>|<div style='width: 170pt;word-wrap: break-word;white-space: normal'>Minimum number of threads in the of frontend worker thread pool for the thrift frontend service</div>|<div style='width: 30pt'>int</div>|<div style='width: 20pt'>1.4.0</div>
@@ -407,8 +411,17 @@ appender.console.name = STDOUT
 appender.console.target = SYSTEM_OUT
 appender.console.layout.type = PatternLayout
 appender.console.layout.pattern = %d{HH:mm:ss.SSS} %p %c: %m%n
-appender.console.filter.a.type = ThresholdFilter
-appender.console.filter.a.level = info
+
+appender.console.filter.1.type = Filters
+
+appender.console.filter.1.a.type = ThresholdFilter
+appender.console.filter.1.a.level = info
+
+# SPARK-34128: Suppress undesirable TTransportException warnings, due to THRIFT-4805
+appender.console.filter.1.b.type = RegexFilter
+appender.console.filter.1.b.regex = .*Thrift error occurred during processing of message.*
+appender.console.filter.1.b.onMatch = deny
+appender.console.filter.1.b.onMismatch = neutral
 
 # Set the default kyuubi-ctl log level to WARN. When running the kyuubi-ctl, the
 # log level for this class is used to overwrite the root logger's log level.

externals/kyuubi-flink-sql-engine/pom.xml

@@ -171,8 +171,7 @@
               <include>org.apache.curator:curator-framework</include>
               <include>org.apache.curator:curator-recipes</include>\
               <include>org.apache.hive:hive-service-rpc</include>
-              <include>org.apache.thrift:libfb303</include>
-              <include>org.apache.thrift:libthrift</include>
+              <include>org.apache.thrift:*</include>
               <include>org.apache.zookeeper:*</include>
             </includes>
           </artifactSet>
@@ -220,13 +219,6 @@
                 <include>org.apache.thrift.**</include>
               </includes>
             </relocation>
-            <relocation>
-              <pattern>com.facebook.fb303</pattern>
-              <shadedPattern>${kyuubi.shade.packageName}.com.facebook.fb303</shadedPattern>
-              <includes>
-                <include>com.facebook.fb303.**</include>
-              </includes>
-            </relocation>
             <relocation>
               <pattern>org.apache.zookeeper</pattern>
               <shadedPattern>${kyuubi.shade.packageName}.org.apache.zookeeper</shadedPattern>