sl/flatten batch non array

.gitignore

@@ -56,4 +56,5 @@ node_modules/
 
 # Misc
 .bsp
-website/.docusaurus
+website/.docusaurus
+null/

build.sbt

@@ -259,6 +259,7 @@ val settings = Seq(
   assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false),
   autoAPIMappings := true,
   pomPostProcess := pomPostFunc,
+  sbtPlugin := false
 )
 ThisBuild / publishMavenStyle := true
 
@@ -361,10 +362,16 @@ testWebsiteDocs := {
   )
 }
 
-sonatypeProjectHosting := Some(
+ThisBuild / sonatypeProjectHosting := Some(
   GitHubHosting("Azure", "SynapseML", "mmlspark-support@microsot.com"))
-homepage := Some(url("https://github.com/Microsoft/SynapseML"))
-developers := List(
+ThisBuild / homepage := Some(url("https://github.com/Microsoft/SynapseML"))
+ThisBuild / scmInfo := Some(
+  ScmInfo(
+    url("https://github.com/Azure/SynapseML"),
+    "scm:git@github.com:Azure/SynapseML.git"
+  )
+)
+ThisBuild / developers := List(
   Developer("mhamilton723", "Mark Hamilton",
     "mmlspark-support@microsoft.com", url("https://github.com/mhamilton723")),
   Developer("imatiach-msft", "Ilya Matiach",
@@ -373,9 +380,9 @@ developers := List(
     "mmlspark-support@microsoft.com", url("https://github.com/drdarshan"))
 )
 
-licenses += ("MIT", url("https://github.com/Microsoft/SynapseML/blob/master/LICENSE"))
+ThisBuild / licenses += ("MIT", url("https://github.com/Microsoft/SynapseML/blob/master/LICENSE"))
 
-credentials += Credentials("Sonatype Nexus Repository Manager",
+ThisBuild / credentials += Credentials("Sonatype Nexus Repository Manager",
   "oss.sonatype.org",
   Secrets.nexusUsername,
   Secrets.nexusPassword)

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/CognitiveServiceBase.scala

@@ -332,10 +332,4 @@ abstract class CognitiveServicesBaseNoHandler(val uid: String) extends Transform
 }
 
 abstract class CognitiveServicesBase(uid: String) extends
-  CognitiveServicesBaseNoHandler(uid) with HasHandler {
-  setDefault(handler -> HandlingUtils.advancedUDF(100)) //scalastyle:ignore magic.number
-
-  override def handlingFunc(client: CloseableHttpClient,
-                            request: HTTPRequestData): HTTPResponseData =
-    getHandler(client, request)
-}
+  CognitiveServicesBaseNoHandler(uid) with HasHandler

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala

@@ -232,7 +232,7 @@ trait BasicAsyncReply extends HasAsyncReply {
                              request: HTTPRequestData): HTTPResponseData = {
     val response = HandlingUtils.advanced(getBackoffs: _*)(client, request)
     if (response.statusLine.statusCode == 202) {
-      val location = new URI(response.headers.filter(_.name == "Operation-Location").head.value)
+      val location = new URI(response.headers.filter(_.name.toLowerCase() == "operation-location").head.value)
       val maxTries = getMaxPollingRetries
       val key = request.headers.find(_.name == "Ocp-Apim-Subscription-Key").map(_.value)
       val it = (0 to maxTries).toIterator.flatMap { _ =>
@@ -255,7 +255,6 @@ trait BasicAsyncReply extends HasAsyncReply {
   }
 }
 
-
 trait HasAsyncReply extends Params {
   val backoffs: IntArrayParam = new IntArrayParam(
     this, "backoffs", "array of backoffs to use in the handler")

cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormOntologyLearner.scala

@@ -53,21 +53,23 @@ class FormOntologyLearner(override val uid: String) extends Estimator[FormOntolo
   }
 
   override def fit(dataset: Dataset[_]): FormOntologyTransformer = {
-    val fromRow = AnalyzeResponse.makeFromRowConverter
-
-    def combine(st1: StructType, st2: StructType): StructType = {
-      FormOntologyLearner.combineDataTypes(st1, st2).asInstanceOf[StructType]
-    }
-
-    val mergedSchema = dataset.toDF()
-      .select(col(getInputCol))
-      .map(extractOntology(fromRow))(Encoders.kryo[StructType])
-      .reduce(combine _)
-
-    new FormOntologyTransformer()
-      .setInputCol(getInputCol)
-      .setOutputCol(getOutputCol)
-      .setOntology(mergedSchema)
+    logFit({
+      val fromRow = AnalyzeResponse.makeFromRowConverter
+
+      def combine(st1: StructType, st2: StructType): StructType = {
+        FormOntologyLearner.combineDataTypes(st1, st2).asInstanceOf[StructType]
+      }
+
+      val mergedSchema = dataset.toDF()
+        .select(col(getInputCol))
+        .map(extractOntology(fromRow))(Encoders.kryo[StructType])
+        .reduce(combine _)
+
+      new FormOntologyTransformer()
+        .setInputCol(getInputCol)
+        .setOutputCol(getOutputCol)
+        .setOntology(mergedSchema)
+    })
   }
 
   override def copy(extra: ParamMap): Estimator[FormOntologyTransformer] = defaultCopy(extra)
@@ -109,11 +111,13 @@ class FormOntologyTransformer(override val uid: String) extends Model[FormOntolo
   }
 
   override def transform(dataset: Dataset[_]): DataFrame = {
-    val fromRow = AnalyzeResponse.makeFromRowConverter
-    val convertToOntologyUDF = UDFUtils.oldUdf(convertToOntology(fromRow) _, getOntology)
+    logTransform[DataFrame]({
+      val fromRow = AnalyzeResponse.makeFromRowConverter
+      val convertToOntologyUDF = UDFUtils.oldUdf(convertToOntology(fromRow) _, getOntology)
 
-    dataset.toDF()
-      .withColumn(getOutputCol, convertToOntologyUDF(col(getInputCol)))
+      dataset.toDF()
+        .withColumn(getOutputCol, convertToOntologyUDF(col(getInputCol)))
+    })
   }
 
   override def transformSchema(schema: StructType): StructType = {