Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Don't reset the dataset type + plug for rm unused columns #6683

Merged
merged 3 commits into from
Aug 24, 2020
Merged

Don't reset the dataset type + plug for rm unused columns #6683

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5292052
Don't reset the type of the dataset
sgugger Aug 24, 2020
ced6d09
Formatting
sgugger Aug 24, 2020
eea7e8d
Update trainer.py
TevenLeScao Aug 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/transformers/trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,8 @@ def __init__(
self.scaler = torch.cuda.amp.GradScaler()

def _remove_unused_columns(self, dataset: "nlp.Dataset", description: Optional[str] = None):
if not self.ars.remove_unused_columns:
TevenLeScao marked this conversation as resolved.
Show resolved Hide resolved
return
# Inspect model forward signature to keep only the arguments it accepts.
signature = inspect.signature(self.model.forward)
signature_columns = list(signature.parameters.keys())
Expand All @@ -255,7 +257,10 @@ def _remove_unused_columns(self, dataset: "nlp.Dataset", description: Optional[s
logger.info(
f"The following columns {dset_description}don't have a corresponding argument in `{self.model.__class__.__name__}.forward` and have been ignored: {', '.join(ignored_columns)}."
)
dataset.set_format(columns=columns)
ds_type = dataset.format["type"]
if ds_type == "python":
ds_type = None
dataset.set_format(type=ds_type, columns=columns)

def _get_train_sampler(self) -> Optional[torch.utils.data.sampler.Sampler]:
if isinstance(self.train_dataset, torch.utils.data.IterableDataset):
Expand Down
9 changes: 9 additions & 0 deletions src/transformers/training_args.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,11 @@ class TrainingArguments:
at the next training step under the keyword argument ``mems``.
run_name (:obj:`str`, `optional`):
A descriptor for the run. Notably used for wandb logging.
remove_unused_columns (:obj:`bool`, `optional`, defaults to :obj:`True`):
If using `nlp.Dataset` datasets, whether or not to automatically remove the columns unused by the model
forward method.

(Note: this behavior is not implemented for :class:`~transformers.TFTrainer` yet.)
"""

output_dir: str = field(
Expand Down Expand Up @@ -234,6 +239,10 @@ class TrainingArguments:
default=None, metadata={"help": "An optional descriptor for the run. Notably used for wandb logging."}
)

remove_unused_columns: Optional[bool] = field(
default=True, metadata={"help": "Remove columns not required by the model when using an nlp.Dataset."}
)

@property
def train_batch_size(self) -> int:
"""
Expand Down