Skip to content

Commit

Permalink
bump v0.4.0
Browse files Browse the repository at this point in the history
  • Loading branch information
gschoeni committed Jan 20, 2024
1 parent e8c77be commit 8d025af
Show file tree
Hide file tree
Showing 4 changed files with 38 additions and 16 deletions.
6 changes: 3 additions & 3 deletions oxen/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions oxen/Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "oxen"
version = "0.3.5"
version = "0.4.0"
edition = "2021"

# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
Expand All @@ -16,7 +16,7 @@ pyo3-log = "0.9.0"
tokio = { version = "1", features = ["full"] }
pyo3-polars = "0.9.0"
serde_json = "1.0.106"
liboxen = "0.9.18"
liboxen = "0.10.6"
# liboxen = { path = "../../rust/Oxen/src/lib" }

[build-dependencies]
Expand Down
22 changes: 22 additions & 0 deletions oxen/python/oxen/dataset.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from oxen import PyDataset
from oxen import RemoteRepo

import os
import polars as pl
Expand All @@ -25,6 +26,27 @@ def load_dataset(repo, paths: Union[str, Sequence[str]], features=None):
return dataset


def download(repo: str, path: str, revision=None, dst=None):
"""
Download files from a remote repo.
Parameters
----------
repo : str
The oxen repository you are loading data from
should be in the format {namespace}/{name}
path : str
The path to the data files
revision : str | None
The commit id or branch name of the version of the data to download
dst : str | None
The path to download the data to.
"""

repo = RemoteRepo(repo)
repo.download(path, revision=revision, dst=dst)


class Dataset:
"""
Dataset object constructs a dataset from a remote or local repo.
Expand Down
22 changes: 11 additions & 11 deletions oxen/python/oxen/remote_repo.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,31 +166,31 @@ def ls(
return self._repo.ls(directory, page_num, page_size).entries

def download(
self, remote_path: str, local_path: Optional[str] = None, revision: str = ""
self, src: str, dst: Optional[str] = None, revision: Optional[str] = None
):
"""
Download a file or directory from the remote repo.
Args:
remote_path: `str`
src: `str`
The path to the remote file
local_path: `str | None`
dst: `str | None`
The path to the local file. If None, will download to
the same path as remote_path
revision: `str`
the same path as `src`
revision: `str | None`
The branch or commit id to download. Defaults to `self.revision`
"""
if local_path is None:
local_path = remote_path
if dst is None:
dst = src
# create parent dir if it does not exist
directory = os.path.dirname(local_path)
directory = os.path.dirname(dst)
if directory and not os.path.exists(directory):
os.makedirs(directory, exist_ok=True)

if revision == "":
self._repo.download(remote_path, local_path, self.revision)
if revision == None:
self._repo.download(src, dst, self.revision)
else:
self._repo.download(remote_path, local_path, revision)
self._repo.download(src, dst, revision)

def add(self, local_path: str, directory: str = ""):
"""
Expand Down

0 comments on commit 8d025af

Please sign in to comment.