Skip to content

Refactor ShardingFilterIterDataPipe into a seperate file (#94095) #987

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

Refactor ShardingFilterIterDataPipe into a seperate file (#94095) #987

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion test/dataloader2/test_dataloader2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import torch.distributed as dist
from torch.testing._internal.common_utils import instantiate_parametrized_tests, parametrize

from torch.utils.data.datapipes.iter.grouping import SHARDING_PRIORITIES
from torch.utils.data.datapipes.iter.sharding import SHARDING_PRIORITIES

from torchdata.dataloader2 import (
communication,
Expand Down
2 changes: 1 addition & 1 deletion test/test_graph.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from _utils._common_utils_for_test import IS_WINDOWS

from torch.utils.data import IterDataPipe
from torch.utils.data.datapipes.iter.grouping import SHARDING_PRIORITIES
from torch.utils.data.datapipes.iter.sharding import SHARDING_PRIORITIES

from torchdata.dataloader2 import DataLoader2, MultiProcessingReadingService, ReadingServiceInterface
from torchdata.dataloader2.graph import find_dps, list_dps, remove_dp, replace_dp, traverse_dps
Expand Down
2 changes: 1 addition & 1 deletion torchdata/dataloader2/reading_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import torch.multiprocessing as mp

from torch.utils.data import DataLoader
from torch.utils.data.datapipes.iter.grouping import SHARDING_PRIORITIES
from torch.utils.data.datapipes.iter.sharding import SHARDING_PRIORITIES

from torchdata._constants import default_dl2_worker_join_timeout_in_s, default_timeout_in_s
from torchdata.dataloader2 import communication
Expand Down
2 changes: 1 addition & 1 deletion torchdata/dataloader2/utils/worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

import torch

from torch.utils.data.datapipes.iter.grouping import SHARDING_PRIORITIES
from torch.utils.data.datapipes.iter.sharding import SHARDING_PRIORITIES

from torchdata.dataloader2 import communication
from torchdata.dataloader2.graph import (
Expand Down
2 changes: 1 addition & 1 deletion torchdata/datapipes/iter/__init__.pyi.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ from torchdata._constants import default_timeout_in_s
from torchdata.datapipes.map import MapDataPipe
from torch.utils.data import DataChunk, IterableDataset, default_collate
from torch.utils.data.datapipes._typing import _DataPipeMeta
from torch.utils.data.datapipes.iter.grouping import SHARDING_PRIORITIES
from torch.utils.data.datapipes.iter.sharding import SHARDING_PRIORITIES

from typing import Any, Callable, Dict, List, Optional, Sequence, TypeVar, Union, Hashable

Expand Down
2 changes: 1 addition & 1 deletion torchdata/datapipes/iter/util/sharding.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

from typing import Iterator, Optional, TypeVar

from torch.utils.data.datapipes.iter.grouping import SHARDING_PRIORITIES
from torch.utils.data.datapipes.iter.sharding import SHARDING_PRIORITIES
from torchdata.datapipes import functional_datapipe
from torchdata.datapipes.iter import IterDataPipe

Expand Down