Skip to content

Commit 11e208a

Browse files
hddongpan3793
authored andcommitted
[KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package
<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/latest/community/contributions.html 2. If the PR is related to an issue in https://github.com/apache/incubator-kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'. 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'. --> ### _Why are the changes needed?_ <!-- Please clarify why the changes are needed. For instance, 1. If you add a feature, you can talk about the use case of it. 2. If you fix a bug, you can clarify why it is a bug. --> 1. move time functions from Rowset to RowSetUtils 2. move schema to spark package ### _How was this patch tested?_ - [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [X] [Run test](https://kyuubi.readthedocs.io/en/latest/develop_tools/testing.html#running-tests) locally before make a pull request Closes #1566 from hddong/move-time-functions. Closes #1565 75de2e8 [hongdongdong] private 8b16217 [hongdongdong] [KYUUBI #1565] Move time functions to RowSetUtils and move schema to spark package Authored-by: hongdongdong <hongdongdong@cmss.chinamobile.com> Signed-off-by: Cheng Pan <chengpan@apache.org>
1 parent 6c235e5 commit 11e208a

File tree

7 files changed

+42
-43
lines changed

7 files changed

+42
-43
lines changed

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,14 @@ import org.apache.spark.sql.types.StructType
2727

2828
import org.apache.kyuubi.{KyuubiSQLException, Utils}
2929
import org.apache.kyuubi.engine.spark.operation.SparkOperation.TIMEZONE_KEY
30+
import org.apache.kyuubi.engine.spark.schema.RowSet
31+
import org.apache.kyuubi.engine.spark.schema.SchemaHelper
3032
import org.apache.kyuubi.engine.spark.session.SparkSessionImpl
3133
import org.apache.kyuubi.operation.{AbstractOperation, FetchIterator, OperationState}
3234
import org.apache.kyuubi.operation.FetchOrientation._
3335
import org.apache.kyuubi.operation.OperationState.OperationState
3436
import org.apache.kyuubi.operation.OperationType.OperationType
3537
import org.apache.kyuubi.operation.log.OperationLog
36-
import org.apache.kyuubi.schema.{RowSet, SchemaHelper}
3738
import org.apache.kyuubi.session.Session
3839

3940
abstract class SparkOperation(opType: OperationType, session: Session)

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/RowSet.scala renamed to externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala

Lines changed: 4 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,25 +15,22 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.kyuubi.schema
18+
package org.apache.kyuubi.engine.spark.schema
1919

2020
import java.nio.ByteBuffer
2121
import java.nio.charset.StandardCharsets
2222
import java.sql.Timestamp
23-
import java.text.SimpleDateFormat
2423
import java.time.{Instant, LocalDate, ZoneId}
25-
import java.time.chrono.IsoChronology
26-
import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder}
27-
import java.time.temporal.ChronoField
28-
import java.util.{Date, Locale}
24+
import java.util.Date
2925

3026
import scala.collection.JavaConverters._
31-
import scala.language.implicitConversions
3227

3328
import org.apache.hive.service.rpc.thrift._
3429
import org.apache.spark.sql.Row
3530
import org.apache.spark.sql.types._
3631

32+
import org.apache.kyuubi.util.RowSetUtils._
33+
3734
object RowSet {
3835

3936
def toTRowSet(
@@ -146,10 +143,6 @@ object RowSet {
146143
ret
147144
}
148145

149-
implicit private def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
150-
ByteBuffer.wrap(bitSet.toByteArray)
151-
}
152-
153146
private def toTColumnValue(
154147
ordinal: Int,
155148
row: Row,
@@ -206,29 +199,6 @@ object RowSet {
206199
}
207200
}
208201

209-
private def createBuilder(): DateTimeFormatterBuilder = {
210-
new DateTimeFormatterBuilder().parseCaseInsensitive()
211-
}
212-
213-
private lazy val dateFormatter = {
214-
createBuilder().appendPattern("yyyy-MM-dd")
215-
.toFormatter(Locale.US)
216-
.withChronology(IsoChronology.INSTANCE)
217-
}
218-
219-
private lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
220-
221-
private lazy val timestampFormatter: DateTimeFormatter = {
222-
createBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
223-
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
224-
.toFormatter(Locale.US)
225-
.withChronology(IsoChronology.INSTANCE)
226-
}
227-
228-
private lazy val simpleTimestampFormatter = {
229-
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
230-
}
231-
232202
/**
233203
* A simpler impl of Spark's toHiveString
234204
*/

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/schema/SchemaHelper.scala renamed to externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelper.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.kyuubi.schema
18+
package org.apache.kyuubi.engine.spark.schema
1919

2020
import java.util.Collections
2121

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import org.apache.spark.sql.types.StructField
2323

2424
import org.apache.kyuubi.Logging
2525
import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.sparkMajorMinorVersion
26-
import org.apache.kyuubi.schema.SchemaHelper
26+
import org.apache.kyuubi.engine.spark.schema.SchemaHelper
2727

2828
/**
2929
* A shim that defines the interface interact with Spark's catalogs

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/RowSetSuite.scala renamed to externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.kyuubi.schema
18+
package org.apache.kyuubi.engine.spark.schema
1919

2020
import java.nio.ByteBuffer
2121
import java.nio.charset.StandardCharsets
@@ -30,7 +30,7 @@ import org.apache.spark.sql.types._
3030
import org.apache.spark.unsafe.types.CalendarInterval
3131

3232
import org.apache.kyuubi.KyuubiFunSuite
33-
import org.apache.kyuubi.schema.RowSet.toHiveString
33+
import org.apache.kyuubi.engine.spark.schema.RowSet.toHiveString
3434

3535
class RowSetSuite extends KyuubiFunSuite {
3636

externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/schema/SchemaHelperSuite.scala renamed to externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/SchemaHelperSuite.scala

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,18 @@
1515
* limitations under the License.
1616
*/
1717

18-
package org.apache.kyuubi.schema
18+
package org.apache.kyuubi.engine.spark.schema
1919

2020
import scala.collection.JavaConverters._
2121

2222
import org.apache.hive.service.rpc.thrift.{TCLIServiceConstants, TTypeId}
2323
import org.apache.spark.sql.types._
2424

2525
import org.apache.kyuubi.KyuubiFunSuite
26+
import org.apache.kyuubi.engine.spark.schema.SchemaHelper._
2627

2728
class SchemaHelperSuite extends KyuubiFunSuite {
2829

29-
import SchemaHelper._
30-
3130
val innerSchema: StructType = new StructType()
3231
.add("a", StringType, nullable = true, "")
3332
.add("b", IntegerType, nullable = true, "")

kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,39 @@
1818
package org.apache.kyuubi.util
1919

2020
import java.nio.ByteBuffer
21+
import java.text.SimpleDateFormat
22+
import java.time.chrono.IsoChronology
23+
import java.time.format.DateTimeFormatter
24+
import java.time.format.DateTimeFormatterBuilder
25+
import java.time.temporal.ChronoField
26+
import java.util.Locale
2127

2228
import scala.language.implicitConversions
2329

24-
object RowSetUtils {
30+
private[kyuubi] object RowSetUtils {
31+
32+
lazy val dateFormatter = {
33+
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd")
34+
.toFormatter(Locale.US)
35+
.withChronology(IsoChronology.INSTANCE)
36+
}
37+
38+
lazy val simpleDateFormatter = new SimpleDateFormat("yyyy-MM-dd", Locale.US)
39+
40+
lazy val timestampFormatter: DateTimeFormatter = {
41+
createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss")
42+
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
43+
.toFormatter(Locale.US)
44+
.withChronology(IsoChronology.INSTANCE)
45+
}
46+
47+
lazy val simpleTimestampFormatter = {
48+
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
49+
}
50+
51+
private def createDateTimeFormatterBuilder(): DateTimeFormatterBuilder = {
52+
new DateTimeFormatterBuilder().parseCaseInsensitive()
53+
}
2554

2655
implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = {
2756
ByteBuffer.wrap(bitSet.toByteArray)

0 commit comments

Comments
 (0)