Merge branch 'release/2.5.0'

tripl-ai · Jan 5, 2021 · 38a24a3 · 38a24a3
2 parents 7efb3b3 + 60cdf73
commit 38a24a3
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 17 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Change Log
 
+## 2.5.0
+
+- bump to Arc 3.7.0.
+
 ## 2.4.1
 
 - set name on Dataframe when `persist=true` to help understand persisted datasets when using the Spark UI.

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -13,7 +13,7 @@ object Dependencies {
   val novocode = "com.novocode" % "junit-interface" % "0.11" % "test"
 
   // arc
-  val arc = "ai.tripl" %% "arc" % "3.4.1" % "provided"
+  val arc = "ai.tripl" %% "arc" % "3.7.0" % "provided"
 
   // spark
   val sparkCatalyst = "org.apache.spark" %% "spark-catalyst" % sparkVersion % "provided"

diff --git a/src/it/scala/ai/tripl/arc/util/TestUtils.scala b/src/it/scala/ai/tripl/arc/util/TestUtils.scala
@@ -55,6 +55,7 @@ object TestUtils {
         inlineSQL: Boolean = true,
         inlineSchema: Boolean = true,
         dropUnsupported: Boolean = false,
+        completionEnvironments: List[String] = List("production", "test"),
     )(implicit spark: SparkSession): ARCContext = {
       val loader = ai.tripl.arc.util.Utils.getContextOrSparkClassLoader
 
@@ -80,6 +81,7 @@ object TestUtils {
         serializableConfiguration=new SerializableConfiguration(spark.sparkContext.hadoopConfiguration),
         userData=collection.mutable.Map.empty,
         resolutionConfig=ConfigFactory.load(),
+        completionEnvironments=completionEnvironments,
       )
     }
 

diff --git a/src/main/scala/ai/tripl/arc/extract/DeltaLakeExtract.scala b/src/main/scala/ai/tripl/arc/extract/DeltaLakeExtract.scala
@@ -39,16 +39,15 @@ class DeltaLakeExtract extends PipelineStagePlugin with JupyterCompleter {
 
   val version = ai.tripl.arc.deltalake.BuildInfo.version
 
-  val snippet = """{
+  def snippet()(implicit arcContext: ARCContext): String = {
+    s"""{
     |  "type": "DeltaLakeExtract",
     |  "name": "DeltaLakeExtract",
-    |  "environments": [
-    |    "production",
-    |    "test"
-    |  ],
+    |  "environments": [${arcContext.completionEnvironments.map { env => s""""${env}""""}.mkString(", ")}],
     |  "inputURI": "hdfs://*.delta",
     |  "outputView": "outputView"
     |}""".stripMargin
+  }
 
   val documentationURI = new java.net.URI(s"${baseURI}/extract/#deltalakeextract")
 

diff --git a/src/main/scala/ai/tripl/arc/load/DeltaLakeLoad.scala b/src/main/scala/ai/tripl/arc/load/DeltaLakeLoad.scala
@@ -33,16 +33,15 @@ class DeltaLakeLoad extends PipelineStagePlugin with JupyterCompleter {
 
   val version = ai.tripl.arc.deltalake.BuildInfo.version
 
-  val snippet = """{
+  def snippet()(implicit arcContext: ARCContext): String = {
+    s"""{
     |  "type": "DeltaLakeLoad",
     |  "name": "DeltaLakeLoad",
-    |  "environments": [
-    |    "production",
-    |    "test"
-    |  ],
+    |  "environments": [${arcContext.completionEnvironments.map { env => s""""${env}""""}.mkString(", ")}],
     |  "inputView": "inputView",
     |  "outputURI": "hdfs://*.delta"
     |}""".stripMargin
+  }
 
   val documentationURI = new java.net.URI(s"${baseURI}/load/#deltalakeload")
 

diff --git a/src/main/scala/ai/tripl/arc/load/DeltaLakeMergeLoad.scala b/src/main/scala/ai/tripl/arc/load/DeltaLakeMergeLoad.scala
@@ -34,13 +34,11 @@ class DeltaLakeMergeLoad extends PipelineStagePlugin with JupyterCompleter {
 
   val version = ai.tripl.arc.deltalake.BuildInfo.version
 
-  val snippet = """{
+  def snippet()(implicit arcContext: ARCContext): String = {
+    s"""{
     |  "type": "DeltaLakeMergeLoad",
     |  "name": "DeltaLakeMergeLoad",
-    |  "environments": [
-    |    "production",
-    |    "test"
-    |  ],
+    |  "environments": [${arcContext.completionEnvironments.map { env => s""""${env}""""}.mkString(", ")}],
     |  "inputView": "inputView",
     |  "outputURI": "hdfs://*.delta",
     |  "condition": "source.primaryKey = target.primaryKey",
@@ -49,6 +47,7 @@ class DeltaLakeMergeLoad extends PipelineStagePlugin with JupyterCompleter {
     |  "whenNotMatchedByTargetInsert": {},
     |  "whenNotMatchedBySourceDelete": {}
     |}""".stripMargin
+  }
 
   val documentationURI = new java.net.URI(s"${baseURI}/load/#deltalakemergeload")
 

diff --git a/src/test/scala/ai/tripl/arc/util/TestUtils.scala b/src/test/scala/ai/tripl/arc/util/TestUtils.scala
@@ -55,6 +55,7 @@ object TestUtils {
         inlineSQL: Boolean = true,
         inlineSchema: Boolean = true,
         dropUnsupported: Boolean = false,
+        completionEnvironments: List[String] = List("production", "test"),
     )(implicit spark: SparkSession): ARCContext = {
       val loader = ai.tripl.arc.util.Utils.getContextOrSparkClassLoader
 
@@ -80,6 +81,7 @@ object TestUtils {
         serializableConfiguration=new SerializableConfiguration(spark.sparkContext.hadoopConfiguration),
         userData=collection.mutable.Map.empty,
         resolutionConfig=ConfigFactory.load(),
+        completionEnvironments=completionEnvironments,
       )
     }
 

diff --git a/version.sbt b/version.sbt
@@ -1 +1 @@
-version := "2.4.1"
+version := "2.5.0"