prefetch mvp #986
prefetch mvp #986
Conversation
Scitator
added
enhancement
Scitator
requested review from
ditwoo,
bagxi,
AlekseySh,
julia-shenshina and
elephantmipt
![github-actions[bot]](https://avatars.githubusercontent.com/in/15368?s=60&v=4){kind=small}
catalyst/data/loader.py
Outdated
| for x in iterable: | ||
| result = func(x) | ||
| result_queue.put(result) | ||
| except BaseException: |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
WPS424 Found except BaseException
| return loader | ||
|
|
||
|
|
||
| class BatchPrefetchLoaderWrapper(ILoaderWrapper): |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
D101 Missing docstring in public class
|
|
||
|
|
||
| class BatchPrefetchLoaderWrapper(ILoaderWrapper): | ||
| def __init__(self, loader: DataLoader, num_prefetches: int = None): |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
D107 Missing docstring in init
| super().__init__(loader) | ||
| self.num_prefetches = num_prefetches or loader.batch_size | ||
|
|
||
| def __iter__(self): |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
D105 Missing docstring in magic method
catalyst/data/loader.py
Outdated
|
|
||
| def predict_batch(self, batch): | ||
| # model inference step | ||
| return self.model(batch[0].to(self.device).view(batch[0].size(0), -1)) |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
E501 line too long (86 > 79 characters)
catalyst/data/loader.py
Outdated
|
|
||
| def predict_batch(self, batch): | ||
| # model inference step | ||
| return self.model(batch[0].to(self.device).view(batch[0].size(0), -1)) |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
W505 doc line too long (86 > 79 characters)
catalyst/data/loader.py
Outdated
| y_hat = self.model(x.view(x.size(0), -1)) | ||
|
|
||
| loss = F.cross_entropy(y_hat, y) | ||
| accuracy01, accuracy03 = metrics.accuracy(y_hat, y, topk=(1, 3)) |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
E501 line too long (80 > 79 characters)
catalyst/data/loader.py
Outdated
| y_hat = self.model(x.view(x.size(0), -1)) | ||
|
|
||
| loss = F.cross_entropy(y_hat, y) | ||
| accuracy01, accuracy03 = metrics.accuracy(y_hat, y, topk=(1, 3)) |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
W505 doc line too long (80 > 79 characters)
catalyst/data/loader.py
Outdated
| loss = F.cross_entropy(y_hat, y) | ||
| accuracy01, accuracy03 = metrics.accuracy(y_hat, y, topk=(1, 3)) | ||
| self.batch_metrics.update( | ||
| {"loss": loss, "accuracy01": accuracy01, "accuracy03": accuracy03} |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
E501 line too long (86 > 79 characters)
catalyst/data/loader.py
Outdated
|
|
||
| batch_size=32 | ||
| loaders = { | ||
| "train": DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=ToTensor()), batch_size=batch_size), |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
W505 doc line too long (124 > 79 characters)
catalyst/data/loader.py
Outdated
| batch_size=32 | ||
| loaders = { | ||
| "train": DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=ToTensor()), batch_size=batch_size), | ||
| "valid": DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=ToTensor()), batch_size=batch_size), |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
E501 line too long (125 > 79 characters)
catalyst/data/loader.py
Outdated
| batch_size=32 | ||
| loaders = { | ||
| "train": DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=ToTensor()), batch_size=batch_size), | ||
| "valid": DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=ToTensor()), batch_size=batch_size), |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
W505 doc line too long (125 > 79 characters)
catalyst/data/loader.py
Outdated
| "train": DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=ToTensor()), batch_size=batch_size), | ||
| "valid": DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=ToTensor()), batch_size=batch_size), | ||
| } | ||
| loaders = {k: BatchPrefetchLoaderWrapper(v) for k, v in loaders.items()} |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
E501 line too long (80 > 79 characters)
catalyst/data/loader.py
Outdated
| "train": DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=ToTensor()), batch_size=batch_size), | ||
| "valid": DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=ToTensor()), batch_size=batch_size), | ||
| } | ||
| loaders = {k: BatchPrefetchLoaderWrapper(v) for k, v in loaders.items()} |
There was a problem hiding this comment.
[pep8] reported by reviewdog 🐶
W505 doc line too long (80 > 79 characters)
Before submitting
catalyst-make-codestyle && catalyst-check-codestyle(pip install -U catalyst-codestyle).make check-docs?pytest .?Description
Related Issue
Type of Change
PR review
Anyone in the community is free to review the PR once the tests have passed.
If we didn't discuss your PR in Github issues there's a high chance it will not be merged.
PS