Skip to content

Commit 4a54b18

Browse files
turboFeiyaooqinn
authored andcommitted
[KYUUBI #2075] Using thread-safe FastDateFormat instead of SimpleDateFormat
### _Why are the changes needed?_ We met below issue when fetching result. ``` Caused by: java.lang.RuntimeException: java.lang.ArrayIndexOutOfBoundsException:46 7022 at sun.util.calendar.BaseCalendar.getCalendarDateFromFixedDate(BaseCalendar.java:453) 7023 at java.util.GregorianCalendar.computeFields(GregorianCalendar.java:2397) 7024 at java.util.GregorianCalendar.computeFields(GregorianCalendar.java:2312) 7025 at java.util.Calendar.setTimeInMillis(Calendar.java:1804) 7026 at java.util.Calendar.setTime(Calendar.java:1770) 7027 at java.text.SimpleDateFormat.format(SimpleDateFormat.java:943) 7028 at java.text.SimpleDateFormat.format(SimpleDateFormat.java:936) 7029 at java.text.DateFormat.format(DateFormat.java:345) 7030 at org.apache.kyuubi.schema.RowSet$.toHiveString(RowSet.scala:245) 7031 at org.apache.kyuubi.schema.RowSet$.$anonfun$toTColumn$3(RowSet.scala:120) 7032 at scala.collection.immutable.List.map(List.scala:290) 7033 at org.apache.kyuubi.schema.RowSet$.toTColumn(RowSet.scala:115) 7034 at org.apache.kyuubi.schema.RowSet$.$anonfun$toColumnBasedSet$1(RowSet.scala:65) 7035 at org.apache.kyuubi.schema.RowSet$.$anonfun$toColumnBasedSet$1$adapted(RowSet.scala:64) 7036 at scala.collection.immutable.List.foreach(List.scala:392) 7037 at org.apache.kyuubi.schema.RowSet$.toColumnBasedSet(RowSet.scala:64) 7038 at org.apache.kyuubi.schema.RowSet$.toTRowSet(RowSet.scala:47) 7039 at org.apache.kyuubi.engine.spark.operation.SparkOperation.getNextRowSet(SparkOperation.scala:183) 7040 at org.apache.kyuubi.operation.OperationManager.getOperationNextRowSet(OperationManager.scala:116) 7041 at org.apache.kyuubi.session.AbstractSession.fetchResults(AbstractSession.scala:197) 7042 at org.apache.kyuubi.service.AbstractBackendService.fetchResults(AbstractBackendService.scala:169) 7043 at org.apache.kyuubi.service.ThriftBinaryFrontendService.FetchResults(ThriftBinaryFrontendService.scala:505) ``` The root cause is that the date time formatter used to convert the result to rowSet is not thread-safe. In this pr, we use thread-safe FastDateFormat instead of SimpleDateFormat. ### _How was this patch tested?_ Existing UT. Closes #2075 from turboFei/thread_safe_fat_stimeformatter. Closes #2075 44ae8fd [Fei Wang] Using thread-safe FastDateFormat instead of SimpleDateFormat Authored-by: Fei Wang <fwang12@ebay.com> Signed-off-by: Kent Yao <yao@apache.org> (cherry picked from commit 5a64e12) Signed-off-by: Kent Yao <yao@apache.org>
1 parent 6f98cc2 commit 4a54b18

File tree

3 files changed

+41
-17
lines changed
  • externals
    • kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema
  • kyuubi-common/src/main/scala/org/apache/kyuubi/util

3 files changed

+41
-17
lines changed

externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ import org.apache.flink.types.Row
3434
import org.apache.hive.service.rpc.thrift._
3535

3636
import org.apache.kyuubi.engine.flink.result.ResultSet
37-
import org.apache.kyuubi.util.RowSetUtils.{dateFormatter, timestampFormatter}
37+
import org.apache.kyuubi.util.RowSetUtils._
3838

3939
object RowSet {
4040

@@ -289,19 +289,19 @@ object RowSet {
289289
"null"
290290

291291
case (d: Int, _: DateType) =>
292-
dateFormatter.format(LocalDate.ofEpochDay(d))
292+
formatLocalDate(LocalDate.ofEpochDay(d))
293293

294294
case (ld: LocalDate, _: DateType) =>
295-
dateFormatter.format(ld)
295+
formatLocalDate(ld)
296296

297297
case (d: Date, _: DateType) =>
298-
dateFormatter.format(d.toInstant)
298+
formatInstant(d.toInstant)
299299

300300
case (ldt: LocalDateTime, _: TimestampType) =>
301-
timestampFormatter.format(ldt)
301+
formatLocalDateTime(ldt)
302302

303303
case (ts: Timestamp, _: TimestampType) =>
304-
timestampFormatter.format(ts.toInstant)
304+
formatInstant(ts.toInstant)
305305

306306
case (decimal: java.math.BigDecimal, _: DecimalType) =>
307307
decimal.toPlainString

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -212,16 +212,16 @@ object RowSet {
212212
"null"
213213

214214
case (d: Date, DateType) =>
215-
simpleDateFormatter.format(d)
215+
formatDate(d)
216216

217217
case (ld: LocalDate, DateType) =>
218-
dateFormatter.format(ld)
218+
formatLocalDate(ld)
219219

220220
case (t: Timestamp, TimestampType) =>
221-
simpleTimestampFormatter.format(t)
221+
formatTimestamp(t)
222222

223223
case (i: Instant, TimestampType) =>
224-
timestampFormatter.withZone(timeZone).format(i)
224+
formatInstant(i, Option(timeZone))
225225

226226
case (bin: Array[Byte], BinaryType) =>
227227
new String(bin, StandardCharsets.UTF_8)

kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,40 +18,64 @@
1818
package org.apache.kyuubi.util
1919

2020
import java.nio.ByteBuffer
21-
import java.text.SimpleDateFormat
21+
import java.sql.Timestamp
22+
import java.time.{Instant, LocalDate, LocalDateTime, ZoneId}
2223
import java.time.chrono.IsoChronology
2324
import java.time.format.DateTimeFormatter
2425
import java.time.format.DateTimeFormatterBuilder
2526
import java.time.temporal.ChronoField
26-
import java.util.Locale
27+
import java.util.{Date, Locale}
2728

2829
import scala.language.implicitConversions
2930

31+
import org.apache.commons.lang3.time.FastDateFormat
32+
3033
private[kyuubi] object RowSetUtils {
3134

32-
lazy val dateFormatter = {
35+
private lazy val dateFormatter = {
3336
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd")
3437
.toFormatter(Locale.US)
3538
.withChronology(IsoChronology.INSTANCE)
3639
}
3740

38-
lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
41+
private lazy val legacyDateFormatter = FastDateFormat.getInstance("yyyy-MM-dd", Locale.US)
3942

40-
lazy val timestampFormatter: DateTimeFormatter = {
43+
private lazy val timestampFormatter: DateTimeFormatter = {
4144
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
4245
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
4346
.toFormatter(Locale.US)
4447
.withChronology(IsoChronology.INSTANCE)
4548
}
4649

47-
lazy val simpleTimestampFormatter = {
48-
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
50+
private lazy val legacyTimestampFormatter = {
51+
FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
4952
}
5053

5154
private def createDateTimeFormatterBuilder(): DateTimeFormatterBuilder = {
5255
new DateTimeFormatterBuilder().parseCaseInsensitive()
5356
}
5457

58+
def formatDate(d: Date): String = {
59+
legacyDateFormatter.format(d)
60+
}
61+
62+
def formatLocalDate(ld: LocalDate): String = {
63+
dateFormatter.format(ld)
64+
}
65+
66+
def formatLocalDateTime(ldt: LocalDateTime): String = {
67+
timestampFormatter.format(ldt)
68+
}
69+
70+
def formatInstant(i: Instant, timeZone: Option[ZoneId] = None): String = {
71+
timeZone.map(timestampFormatter.withZone(_).format(i))
72+
.getOrElse(timestampFormatter.format(i))
73+
}
74+
75+
def formatTimestamp(t: Timestamp): String = {
76+
legacyTimestampFormatter.format(t)
77+
}
78+
5579
implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
5680
ByteBuffer.wrap(bitSet.toByteArray)
5781
}

0 commit comments

Comments
 (0)