Skip to content

Commit

Permalink
Couple last fixes (#19)
Browse files Browse the repository at this point in the history 
* Couple last fixes

* fmt

* more fixes
  • Loading branch information
@mgyucht
mgyucht committed Sep 4, 2023
1 parent ac8e18b commit 48629ef
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 49 deletions.
23 changes: 11 additions & 12 deletions databricks-dbutils-scala/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@
<version>0.1.0</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>com.databricks</groupId>
<artifactId>databricks-dbutils-scala_${scala.version}</artifactId>
<artifactId>databricks-dbutils-scala_${scala.major}.${scala.minor}</artifactId>
<version>0.1.0</version>
<name>DBUtils for Scala</name>
<description>DBUtils for Scala simplifies interacting with various components of Databricks, such as the Databricks File
Expand Down Expand Up @@ -55,15 +54,17 @@
<jackson.version>2.15.2</jackson.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<!-- Scala version is overridden by profiles -->
<scala.version>2.12.10</scala.version>
<scala.major>2</scala.major>
<!-- Scala version is overridden by profiles but is needed here for the pom to be valid -->
<scala.minor>12</scala.minor>
<scala.patch>10</scala.patch>
</properties>
<dependencies>
<!-- Scala Standard Library -->
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<version>${scala.major}.${scala.minor}.${scala.patch}</version>
</dependency>
<!-- Nullable annotation -->
<dependency>
Expand Down Expand Up @@ -113,11 +114,11 @@
<artifactId>scala-maven-plugin</artifactId>
<version>4.4.0</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<scalaVersion>${scala.major}.${scala.minor}.${scala.patch}</scalaVersion>
<compilerPlugins>
<compilerPlugin>
<groupId>com.artima.supersafe</groupId>
<artifactId>supersafe_${scala.version}</artifactId>
<artifactId>supersafe_${scala.major}.${scala.minor}.${scala.patch}</artifactId>
<version>1.1.12</version>
</compilerPlugin>
</compilerPlugins>
Expand Down Expand Up @@ -169,7 +170,7 @@
<goals>
<goal>sign</goal>
</goals>
<phase>verify</phase>
<phase>deploy</phase>
</execution>
</executions>
</plugin>
Expand Down Expand Up @@ -255,9 +256,6 @@
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<properties>
<scala.version>2.12.10</scala.version>
</properties>
<dependencies>
<!-- ScalaTest 2.12 -->
<dependency>
Expand Down Expand Up @@ -320,7 +318,8 @@
<profile>
<id>scala-2.13</id>
<properties>
<scala.version>2.13.5</scala.version>
<scala.minor>13</scala.minor>
<scala.patch>5</scala.patch>
</properties>
<dependencies>
<!-- ScalaTest 2.13 -->
Expand Down
9 changes: 9 additions & 0 deletions databricks-dbutils-scala/src/main/assembly/test-jar-with-deps.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@
<useProjectAttachments>true</useProjectAttachments>
<!-- Unzip dependencies and merge classes into the assembly jar (otherwise, the assembly jar will contain jars) -->
<unpack>true</unpack>
<unpackOptions>
<excludes>
<exclude>**/META-INF/versions/9/**</exclude>
</excludes>
</unpackOptions>
<!-- Include test dependencies, as we plan to run the tests from the test jar in Databricks -->
<scope>test</scope>
<!-- To minimize conflicts when running tests on Databricks, we should only include jars here that
Expand All @@ -27,6 +32,10 @@
<!-- These test libraries are not part of DBR. -->
<include>org.scalactic:*</include>
<include>org.scalatest:*</include>
<!-- Mockito deps -->
<include>org.mockito:*</include>
<include>org.objenesis:*</include>
<include>net.bytebuddy:*</include>
</includes>
</dependencySet>
</dependencySets>
Expand Down
2 changes: 2 additions & 0 deletions ...icks-dbutils-scala/src/main/scala/com/databricks/sdk/scala/dbutils/ProxyDBUtilsImpl.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,7 @@ class ProxyDBUtilsImpl private[dbutils] (baseObj: AnyRef) extends DBUtils {
"ls" ->
new MethodCallAdapter(convertResult = { p =>
p.asInstanceOf[Seq[AnyRef]].map { p =>
// Note: modificationTime is not in DBR < 10.4 LTS.
FileInfo(p.getField("path"), p.getField("name"), p.getField("size"), p.getField("modificationTime"))
}
}),
Expand Down Expand Up @@ -198,6 +199,7 @@ class ProxyDBUtilsImpl private[dbutils] (baseObj: AnyRef) extends DBUtils {
override val credentials: DatabricksCredentialUtils =
getProxyInstance[DatabricksCredentialUtils](baseObj.getField("credentials"))
override val jobs: JobsUtils = new ProxyJobsUtils(baseObj.getField("jobs"))
// Not in DBR 7.3
override val data: DataUtils = getProxyInstance[DataUtils](baseObj.getField("data"))
}

Expand Down
8 changes: 6 additions & 2 deletions ...bricks-dbutils-scala/src/main/scala/com/databricks/sdk/scala/dbutils/SdkDBUtilsImpl.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ private object SdkDbfsUtilsImpl {
def unsupportedMethod(methodName: String): Nothing =
throw new UnsupportedOperationException(s"Method $methodName is not supported in the SDK version of DBUtils.")

def unsupportedField(methodName: String): Nothing =
throw new UnsupportedOperationException(s"Field $methodName is not supported in the SDK version of DBUtils.")
def unsupportedField(fieldName: String): Nothing =
throw new UnsupportedOperationException(s"Field $fieldName is not supported in the SDK version of DBUtils.")
}

/** Help is a no-op in the SDK version of DBUtils. */
Expand Down Expand Up @@ -60,6 +60,10 @@ private class SdkDbfsUtils(w: WorkspaceClient) extends DbfsUtils with NoHelp {
}

private def mv(from: String, to: String, recurse: Boolean, delete: Boolean): Boolean = {
if (from == to) {
return true
}

val inputStream = w.files().download(from).getContents
try {
w.files().upload(new UploadRequest().setFilePath(to).setContents(inputStream))
Expand Down
90 changes: 55 additions & 35 deletions examples/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,64 @@
<artifactId>databricks-dbutils-scala-examples</artifactId>
<version>0.1.0</version>
<name>DBUtils for Scala Examples</name>
<properties>
<scala.major>2</scala.major>
<scala.minor>12</scala.minor>
<scala.patch>10</scala.patch>
</properties>
<dependencies>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-reload4j</artifactId>
<version>2.0.7</version>
</dependency>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>databricks-dbutils-scala_${scala.major}.${scala.minor}</artifactId>
<version>0.1.0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.4.0</version>
<configuration>
<scalaVersion>${scala.major}.${scala.minor}.${scala.patch}</scalaVersion>
</configuration>
<executions>
<execution>
<id>scala-compile</id>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.0.0</version>
<configuration>
<mainClass>com.databricks.sdk.scala.dbutils.examples.Example</mainClass>
<!-- You can also include arguments if needed -->
<!--
<arguments>
<argument>arg1</argument>
<argument>arg2</argument>
</arguments>
-->
</configuration>
</plugin>
</plugins>
</build>
<profiles>
<profile>
<id>scala-2.12</id>
<dependencies>
<dependency>
<groupId>com.databricks</groupId>
<artifactId>databricks-dbutils-scala_2.12.10</artifactId>
<version>0.1.0</version>
</dependency>
</dependencies>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
<build>
<plugins>
<plugin>
Expand All @@ -43,36 +84,15 @@
</scala>
</configuration>
</plugin>
<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
<version>4.4.0</version>
<executions>
<execution>
<id>scala-compile</id>
<goals>
<goal>compile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
<version>3.0.0</version>
<configuration>
<mainClass>com.databricks.sdk.scala.dbutils.examples.Example</mainClass>
<!-- You can also include arguments if needed -->
<!--
<arguments>
<argument>arg1</argument>
<argument>arg2</argument>
</arguments>
-->
</configuration>
</plugin>
</plugins>
</build>
</profile>
<profile>
<id>scala-2.13</id>
<properties>
<scala.minor>13</scala.minor>
<scala.patch>5</scala.patch>
</properties>
</profile>
</profiles>
</project>

0 comments on commit 48629ef

Please sign in to comment.