Skip to content

Commit

Permalink
Allow features to be immutable
Browse files Browse the repository at this point in the history
PiperOrigin-RevId: 502624945
  • Loading branch information
T5 Team authored and t5-copybara committed Jan 17, 2023
1 parent 3d35c59 commit f0cf9e8
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 1 deletion.
1 change: 1 addition & 0 deletions setup.py
Expand Up @@ -46,6 +46,7 @@
'absl-py',
'babel',
'editdistance',
'immutabledict',
'gin-config',
'mesh-tensorflow[transformer]>=0.1.13',
'nltk',
Expand Down
3 changes: 2 additions & 1 deletion t5/data/dataset_providers.py
Expand Up @@ -22,6 +22,7 @@
between.
"""

from collections.abc import Mapping
import re

import seqio
Expand Down Expand Up @@ -88,7 +89,7 @@ def __init__(self,
raise ValueError("output_features must be non-empty.")
if output_features is None:
output_features = seqio.Feature(utils.get_default_vocabulary())
if isinstance(output_features, dict):
if isinstance(output_features, Mapping):
pass
elif isinstance(output_features, seqio.Feature):
output_features = {k: output_features for k in _DEFAULT_FEATURE_KEYS}
Expand Down
10 changes: 10 additions & 0 deletions t5/data/dataset_providers_test.py
Expand Up @@ -16,6 +16,7 @@
import os

from absl.testing import absltest
import immutabledict
import seqio
from seqio import test_utils
from t5.data import dataset_providers
Expand Down Expand Up @@ -53,6 +54,15 @@ def test_tfds_task(self):
"t5_tfds_task", dataset_providers.TfdsTask, tfds_name="fake:0.0.0")
self.verify_task_matches_fake_datasets("t5_tfds_task", use_cached=False)

def test_immutabledict_features(self):
_add_t5_task(
"t5_tfds_task", dataset_providers.TfdsTask, tfds_name="fake:0.0.0",
output_features=immutabledict.immutabledict({
"inputs": seqio.Feature(test_utils.sentencepiece_vocab()),
"targets": seqio.Feature(test_utils.sentencepiece_vocab())
}))
self.verify_task_matches_fake_datasets("t5_tfds_task", use_cached=False)

def test_function_task(self):
_add_t5_task(
"t5_fn_task",
Expand Down

0 comments on commit f0cf9e8

Please sign in to comment.