Skip to content
Permalink
Browse files

Update Scala version to 2.12 and Arrow to 0.12 (#72)

  • Loading branch information...
jaewanbahk authored and icexelloss committed Aug 6, 2019
1 parent eda21fa commit 40fd887bebbf5a5d42365b914a7c99a665ccf8c7
@@ -20,4 +20,4 @@ dist: clean-pyc
sbt "set test in assembly := {}" clean assembly
cd python; \
find . -mindepth 2 -name '*.py' -print | \
zip ../target/scala-2.11/flint-assembly-$(VERSION)-SNAPSHOT.jar -@
zip ../target/flint-assembly-$(VERSION)-SNAPSHOT.jar -@
@@ -31,7 +31,7 @@ lazy val formattingPreferences = {
lazy val compilationSettings = scalariformSettings ++ Seq(
version := "0.6.0-SNAPSHOT",
organization := "com.twosigma",
scalaVersion := "2.11.8",
scalaVersion := "2.12.8",
assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
javacOptions ++= Seq("-source", "1.7", "-target", "1.7"),
compileOrder in Compile := CompileOrder.JavaThenScala,
@@ -54,16 +54,16 @@ lazy val compilationSettings = scalariformSettings ++ Seq(
)

lazy val versions = new {
val play_json = "2.5.0"
val play_json = "2.7.4"
val commons_math = "3.5"
val joda_time = "2.9.4"
val httpclient = "4.3.2" // Note that newer versions need to be configured
val spark = sys.props.getOrElse("spark.version", default = "2.4.3")
val scalatest = "2.2.4"
val scalatest = "3.0.8"
val scalacheck = "1.12.6"
val grizzled_slf4j = "1.3.0"
val arrow = "0.10.0"
val jackson_module = "2.7.2"
val arrow = "0.12.0"
val jackson_module = "2.7.9"
}

lazy val lazyDependencies = new {
@@ -105,7 +105,7 @@ object ArrowConverters {

var closed = false

context.addTaskCompletionListener { _ =>
context.addTaskCompletionListener[Unit] { _ =>
if (!closed) {
root.close()
allocator.close()
@@ -519,7 +519,7 @@ class OrderedRDD[K: ClassTag, V: ClassTag](
new OrderedRDD[K, Array[V]](sc, rangeSplits, Seq(new OneToOneDependency(self)))(
(p, tc) => {
val iter = new SummarizeByKeyIterator(iterator(p, tc), skFn, new RowsSummarizer[V])
tc.addTaskCompletionListener((_) => iter.close())
tc.addTaskCompletionListener[Unit]((_) => iter.close())
iter.map { case (k, (_, v)) => (k, v) }
}
)
@@ -543,7 +543,7 @@ class OrderedRDD[K: ClassTag, V: ClassTag](
new OrderedRDD[K, (SK, V2)](sc, rangeSplits, Seq(new OneToOneDependency(self)))(
(p, tc) => {
val iter = new SummarizeByKeyIterator(iterator(p, tc), skFn, summarizer)
tc.addTaskCompletionListener((_) => iter.close())
tc.addTaskCompletionListener[Unit]((_) => iter.close())
iter
}
)
@@ -1946,7 +1946,7 @@ class TimeSeriesRDDImpl private[timeseries] (
val newOrdd = orderedRdd.mapPartitionsWithIndexOrdered {
case (_, rows) =>
val allocator = new RootAllocator(Long.MaxValue)
TaskContext.get().addTaskCompletionListener { _ =>
TaskContext.get().addTaskCompletionListener[Unit] { _ =>
allocator.close()
}

@@ -1 +1 @@
0.2.0
0.6.0

0 comments on commit 40fd887

Please sign in to comment.
You can’t perform that action at this time.