Skip to content

Commit

Permalink
comments added
Browse files Browse the repository at this point in the history
  • Loading branch information
AbhinavTuli committed Feb 19, 2021
1 parent 4ecd521 commit f8253ed
Show file tree
Hide file tree
Showing 3 changed files with 6 additions and 8 deletions.
1 change: 1 addition & 0 deletions hub/api/compute_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import numpy as np


# a list of Datasets or DatasetViews or Tensorviews that supports compute operation
class ComputeList:
# Doesn't support further get item operations currently
def __init__(self, ls):
Expand Down
7 changes: 5 additions & 2 deletions hub/api/sharded_datasetview.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@
"""

from collections.abc import Iterable
import numpy as np
from hub.exceptions import AdvancedSlicingNotSupported
from hub.api.dataset_utils import slice_split
from hub.api.compute_list import ComputeList

Expand Down Expand Up @@ -71,10 +69,13 @@ def __getitem__(self, slice_):
subpath, slice_list = slice_split(slice_)
slice_list = slice_list or [slice(0, self.num_samples)]
if isinstance(slice_list[0], int):
# if integer it fetches the data from the corresponding dataset
slice_list, shard_id = self.slicing(slice_list)
slice_ = slice_list + [subpath] if subpath else slice_list
return self.datasets[shard_id][slice_]
else:
# if slice it finds all the corresponding datasets included in the slice and generates tensorviews or datasetviews (depending on slice)
# these views are stored in a ComputeList, calling compute on which will fetch data from all corresponding datasets and return a single result
results = []
cur_index = slice_list[0].start or 0
cur_index = cur_index + self.num_samples if cur_index < 0 else cur_index
Expand All @@ -101,10 +102,12 @@ def __setitem__(self, slice_, value) -> None:
subpath, slice_list = slice_split(slice_)
slice_list = slice_list or [slice(0, self.num_samples)]
if isinstance(slice_list[0], int):
# if integer it assigns the data to the corresponding dataset
slice_list, shard_id = self.slicing(slice_list)
slice_ = slice_list + [subpath] if subpath else slice_list
self.datasets[shard_id][slice_] = value
else:
# if slice it finds all the corresponding datasets and assigns slices of the value one by one
cur_index = slice_list[0].start or 0
cur_index = cur_index + self.num_samples if cur_index < 0 else cur_index
cur_index = max(cur_index, 0)
Expand Down
6 changes: 0 additions & 6 deletions hub/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,12 +261,6 @@ def __init__(self):
super(HubException, self).__init__(message=message)


class AdvancedSlicingNotSupported(HubException):
def __init__(self):
message = "Advanced slicing is not supported, only support index"
super(HubException, self).__init__(message=message)


class NotZarrFolderException(Exception):
pass

Expand Down

0 comments on commit f8253ed

Please sign in to comment.