Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions NOTICE
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,7 @@ Copyright (2022) Databricks, Inc.


This Software includes software developed at Databricks (https://www.databricks.com/) and its use is subject to the included LICENSE file.

--------------------------------------------------------------------------------

This Software includes adapted class from https://github.com/benwatson528/intellij-avro-parquet-plugin.
33 changes: 31 additions & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,31 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>io.delta.sharing.java</groupId>
<artifactId>DeltaSharingJavaConnector</artifactId>
<groupId>com.databricks.labs</groupId>
<artifactId>delta-sharing-java-connector</artifactId>
<version>0.1.0-SNAPSHOT</version>

<developers>
<developer>
<id>milos-colic</id>
<name>Milos Colic</name>
<email>milos.colic@databricks.com</email>
<organization>Databricks</organization>
<roles>
<role>Solutions Architect</role>
</roles>
</developer>
<developer>
<id>vuong-nguyen</id>
<name>Vuong Nguyen</name>
<email>vuong.nguyen@databricks.com</email>
<organization>Databricks</organization>
<roles>
<role>Solutions Architect</role>
</roles>
</developer>
</developers>

<properties>
<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
Expand All @@ -23,6 +44,7 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.12.3</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.parquet</groupId>
Expand All @@ -33,11 +55,13 @@
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>1.12.3</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
<version>1.2.1</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>commons-httpclient</groupId>
Expand All @@ -49,6 +73,7 @@
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>junit</groupId>
Expand All @@ -60,21 +85,25 @@
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.5.13</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.12.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.12</artifactId>
<version>3.2.1</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import java.nio.file.OpenOption;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.FileAttribute;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
Expand All @@ -43,15 +44,14 @@
* available local copy in the temp directory.
* <p/>
*
* @author Milos Colic
* @since 0.1.0
*/
public class DeltaSharing {
private DeltaSharingProfileProvider profileProvider;
private DeltaSharingRestClient httpClient;
private Path checkpointPath;
private Path tempDir;
private Map<String, DeltaTableMetadata> metadataMap;
private final DeltaSharingProfileProvider profileProvider;
private final DeltaSharingRestClient httpClient;
private final Path checkpointPath;
private final Path tempDir;
private final Map<String, DeltaTableMetadata> metadataMap;

/**
* Getter for {@link DeltaSharing#profileProvider}.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
* creation will fail in case of breaking the constraints.
* <p/>
*
* @author Milos Colic
* @see DeltaSharingProfileAdaptor DeltaSharingProfileAdaptor used for object
* mapping when parsing JSON.
* @since 0.1.0
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
* @implNote Suppress is added because of all the getters and setters are required to be explicitly
* created for Jackson to parse JSONs correctly. However warnings are shown since getters
* and setters are not explicitly tests.
* @author Milos Colic
* @since 0.1.0
*/
public class DeltaSharingProfileAdaptor {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,9 @@
import org.apache.parquet.io.SeekableInputStream;

/**
* Parquet InputFile with a local java.nio.Path. Adapted from https://github.com/haifengl/smile This
* Parquet InputFile with a local java.nio.Path. Adapted from https://github.com/benwatson528/intellij-avro-parquet-plugin/blob/master/src/main/java/uk/co/hadoopathome/intellij/viewer/fileformat/LocalInputFile.java This
* class is required to instantiate {@link org.apache.parquet.avro.AvroParquetReader} instances.
* @implNote https://github.com/benwatson528/intellij-avro-parquet-plugin is licenced under Apache 2.0.
*/
public class LocalInputFile implements InputFile {
/** Local file object. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@
* constitute a table.
* <p/>
*
* @author Milos Colic
* @since 1.0.0
* @since 0.1.0
*/
public class TableReader<T> {
private Queue<ParquetReader<T>> readers;
Expand Down