Skip to content

Commit

Permalink
[SPARK-26546][SQL] Caching of java.time.format.DateTimeFormatter
Browse files Browse the repository at this point in the history
## What changes were proposed in this pull request?

Added a cache for  java.time.format.DateTimeFormatter instances with keys consist of pattern and locale. This should allow to avoid parsing of timestamp/date patterns each time when new instance of `TimestampFormatter`/`DateFormatter` is created.

## How was this patch tested?

By existing test suites `TimestampFormatterSuite`/`DateFormatterSuite` and `JsonFunctionsSuite`/`JsonSuite`.

Closes #23462 from MaxGekk/time-formatter-caching.

Lead-authored-by: Maxim Gekk <max.gekk@gmail.com>
Co-authored-by: Maxim Gekk <maxim.gekk@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
  • Loading branch information
2 people authored and HyukjinKwon committed Jan 10, 2019
1 parent 1a47233 commit 73c7b12
Show file tree
Hide file tree
Showing 3 changed files with 40 additions and 15 deletions.
Expand Up @@ -36,7 +36,7 @@ class Iso8601DateFormatter(
locale: Locale) extends DateFormatter with DateTimeFormatterHelper {

@transient
private lazy val formatter = buildFormatter(pattern, locale)
private lazy val formatter = getOrCreateFormatter(pattern, locale)
private val UTC = ZoneId.of("UTC")

private def toInstant(s: String): Instant = {
Expand Down
Expand Up @@ -23,9 +23,46 @@ import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, ResolverSt
import java.time.temporal.{ChronoField, TemporalAccessor, TemporalQueries}
import java.util.Locale

import com.google.common.cache.CacheBuilder

import org.apache.spark.sql.catalyst.util.DateTimeFormatterHelper._

trait DateTimeFormatterHelper {
protected def toInstantWithZoneId(temporalAccessor: TemporalAccessor, zoneId: ZoneId): Instant = {
val localTime = if (temporalAccessor.query(TemporalQueries.localTime) == null) {
LocalTime.ofNanoOfDay(0)
} else {
LocalTime.from(temporalAccessor)
}
val localDate = LocalDate.from(temporalAccessor)
val localDateTime = LocalDateTime.of(localDate, localTime)
val zonedDateTime = ZonedDateTime.of(localDateTime, zoneId)
Instant.from(zonedDateTime)
}

// Gets a formatter from the cache or creates new one. The buildFormatter method can be called
// a few times with the same parameters in parallel if the cache does not contain values
// associated to those parameters. Since the formatter is immutable, it does not matter.
// In this way, synchronised is intentionally omitted in this method to make parallel calls
// less synchronised.
// The Cache.get method is not used here to avoid creation of additional instances of Callable.
protected def getOrCreateFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
val key = (pattern, locale)
var formatter = cache.getIfPresent(key)
if (formatter == null) {
formatter = buildFormatter(pattern, locale)
cache.put(key, formatter)
}
formatter
}
}

protected def buildFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
private object DateTimeFormatterHelper {
val cache = CacheBuilder.newBuilder()
.maximumSize(128)
.build[(String, Locale), DateTimeFormatter]()

def buildFormatter(pattern: String, locale: Locale): DateTimeFormatter = {
new DateTimeFormatterBuilder()
.parseCaseInsensitive()
.appendPattern(pattern)
Expand All @@ -38,16 +75,4 @@ trait DateTimeFormatterHelper {
.withChronology(IsoChronology.INSTANCE)
.withResolverStyle(ResolverStyle.STRICT)
}

protected def toInstantWithZoneId(temporalAccessor: TemporalAccessor, zoneId: ZoneId): Instant = {
val localTime = if (temporalAccessor.query(TemporalQueries.localTime) == null) {
LocalTime.ofNanoOfDay(0)
} else {
LocalTime.from(temporalAccessor)
}
val localDate = LocalDate.from(temporalAccessor)
val localDateTime = LocalDateTime.of(localDate, localTime)
val zonedDateTime = ZonedDateTime.of(localDateTime, zoneId)
Instant.from(zonedDateTime)
}
}
Expand Up @@ -51,7 +51,7 @@ class Iso8601TimestampFormatter(
timeZone: TimeZone,
locale: Locale) extends TimestampFormatter with DateTimeFormatterHelper {
@transient
private lazy val formatter = buildFormatter(pattern, locale)
private lazy val formatter = getOrCreateFormatter(pattern, locale)

private def toInstant(s: String): Instant = {
val temporalAccessor = formatter.parse(s)
Expand Down

0 comments on commit 73c7b12

Please sign in to comment.