Merge branch 'main' of github.com:dask-contrib/dask-deltatable into f…

…eature/dependency-fix

.github/workflows/pre-commit.yaml

@@ -3,9 +3,12 @@ name: Python Style Check
 on: [push,pull_request]
 
 jobs:
-  pre-commit:
+  checks:
+    name: pre-commit hooks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
-      - uses: pre-commit/action@v2.0.0
+      - uses: actions/checkout@v3.5.2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+      - uses: pre-commit/action@v3.0.0

.pre-commit-config.yaml

@@ -5,7 +5,7 @@ repos:
       - id: black
         language_version: python3
   - repo: https://github.com/pycqa/isort
-    rev: 5.7.0
+    rev: 5.12.0
     hooks:
       - id: isort
         args:

LICENSE

@@ -0,0 +1,31 @@
+BSD 3-Clause License
+
+Copyright (c) 2023, Dask contributors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+* Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

dask_deltatable/core.py

@@ -28,13 +28,18 @@ def __init__(
         columns: List[str],
         datetime: Optional[str] = None,
         storage_options: Dict[str, str] = None,
+        delta_storage_options: Dict[str, str] = None,
     ) -> None:
         self.path: str = path
         self.version: int = version
         self.columns = columns
         self.datetime = datetime
         self.storage_options = storage_options
-        self.dt = DeltaTable(table_uri=self.path, version=self.version)
+        self.dt = DeltaTable(
+            table_uri=self.path,
+            version=self.version,
+            storage_options=delta_storage_options,
+        )
         self.fs, self.fs_token, _ = get_fs_token_paths(
             path, storage_options=storage_options
         )
@@ -201,6 +206,7 @@ def read_delta_table(
     columns: List[str] = None,
     storage_options: Dict[str, str] = None,
     datetime: str = None,
+    delta_storage_options: Dict[str, str] = None,
     **kwargs,
 ):
     """
@@ -238,7 +244,9 @@ def read_delta_table(
     columns: None or list(str)
         Columns to load. If None, loads all.
     storage_options : dict, default None
-        Key/value pairs to be passed on to the file-system backend, if any.
+        Key/value pairs to be passed on to the fsspec backend, if any.
+    delta_storage_options : dict, default None
+        Key/value pairs to be passed on to the delta-rs filesystem, if any.
     kwargs: dict,optional
         Some most used parameters can be passed here are:
         1. schema
@@ -285,13 +293,17 @@ def read_delta_table(
             columns=columns,
             storage_options=storage_options,
             datetime=datetime,
+            delta_storage_options=delta_storage_options,
         )
         resultdf = dtw.read_delta_table(columns=columns, **kwargs)
     return resultdf
 
 
 def read_delta_history(
-    path: str, limit: Optional[int] = None, storage_options: Dict[str, str] = None
+    path: str,
+    limit: Optional[int] = None,
+    storage_options: Dict[str, str] = None,
+    delta_storage_options: Dict[str, str] = None,
 ) -> dd.core.DataFrame:
     """
     Run the history command on the DeltaTable.
@@ -313,7 +325,11 @@ def read_delta_history(
     """
 
     dtw = DeltaTableWrapper(
-        path=path, version=None, columns=None, storage_options=storage_options
+        path=path,
+        version=None,
+        columns=None,
+        storage_options=storage_options,
+        delta_storage_options=delta_storage_options,
     )
     return dtw.history(limit=limit)
 
@@ -323,6 +339,7 @@ def vacuum(
     retention_hours: int = 168,
     dry_run: bool = True,
     storage_options: Dict[str, str] = None,
+    delta_storage_options: Dict[str, str] = None,
 ) -> None:
     """
     Run the Vacuum command on the Delta Table: list and delete
@@ -344,6 +361,10 @@ def vacuum(
     """
 
     dtw = DeltaTableWrapper(
-        path=path, version=None, columns=None, storage_options=storage_options
+        path=path,
+        version=None,
+        columns=None,
+        storage_options=storage_options,
+        delta_storage_options=delta_storage_options,
     )
     return dtw.vacuum(retention_hours=retention_hours, dry_run=dry_run)

setup.py

@@ -11,7 +11,7 @@
     description="Dask + Delta Table ",
     maintainer="rajagurunath",
     maintainer_email="gurunathrajagopal@gmail.com",
-    license="MIT",
+    license="BSD-3-Clause",
     packages=["dask_deltatable"],
     long_description=long_description,
     long_description_content_type="text/markdown",