From 11629d449ee6d65b5434a82a077e168c0c398aad Mon Sep 17 00:00:00 2001 From: Tim Semenov Date: Sun, 17 Aug 2025 08:26:29 -0700 Subject: [PATCH] Fix `c4_test`. PiperOrigin-RevId: 796108166 --- .github/workflows/pytest-template.yml | 32 ++++++++++-- setup.py | 49 ++++++++++++------- .../core/lazy_imports_lib_test.py | 8 +-- 3 files changed, 61 insertions(+), 28 deletions(-) diff --git a/.github/workflows/pytest-template.yml b/.github/workflows/pytest-template.yml index a466af87e1d..a90f2216ff9 100644 --- a/.github/workflows/pytest-template.yml +++ b/.github/workflows/pytest-template.yml @@ -52,10 +52,7 @@ jobs: # Run tests # Ignores: - # * Nsynth is run in isolation due to dependency conflict (crepe). - # * Lsun tests is disabled because the tensorflow_io used in open-source - # is linked to static libraries compiled again specific TF version, which - # makes test fails with linking error (libtensorflow_io_golang.so). + # * c4, lsun, nsynth are run separately to avoid dependency conflicts. # * imagenet2012_corrupted requires imagemagick binary. # * import_without_tf_test.py, because the test relies on TensorFlow not being imported. # * github_api is run separately to not overuse API quota. @@ -65,8 +62,9 @@ jobs: - name: Run core tests run: | pytest --durations=100 -vv -n auto --shard-id=$((${{ matrix.shard-id }} - 1)) --num-shards=${{ matrix.num-shards }} \ - --ignore="tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py" \ + --ignore-glob="tensorflow_datasets/text/c4*_test.py" \ --ignore="tensorflow_datasets/image/lsun_test.py" \ + --ignore="tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py" \ --ignore="tensorflow_datasets/datasets/imagenet2012_corrupted/imagenet2012_corrupted_dataset_builder_test.py" \ --ignore="tensorflow_datasets/scripts/documentation/build_api_docs_test.py" \ --ignore="tensorflow_datasets/import_without_tf_test.py" \ @@ -86,3 +84,27 @@ jobs: retry_on: timeout command: | pytest -vv -o faulthandler_timeout=10 tensorflow_datasets/translate/wmt19_test.py + + # Run `c4` dataset tests separately. + - name: Run c4 dataset tests + if: ${{ matrix.shard-id == 1 }} + run: | + pip install .[c4] + pytest -vv -n auto \ + tensorflow_datasets/text/c4*_test.py + + # Run `lsun` dataset tests separately. + - name: Run lsun dataset tests + if: ${{ matrix.shard-id == 2 }} + run: | + pip install .[lsun] + pytest -vv -n auto \ + tensorflow_datasets/image/lsun_test.py + + # Run `nsynth` dataset tests separately. + - name: Run nsynth dataset tests + if: ${{ matrix.shard-id == 3 }} + run: | + pip install .[nsynth] + pytest -vv -n auto \ + tensorflow_datasets/datasets/nsynth/nsynth_dataset_builder_test.py diff --git a/setup.py b/setup.py index 4d7170990b2..6cd5404560e 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,6 @@ 'pytest-shard', 'pytest-xdist', # Lazy-deps required by core - # TODO(b/418761065): Update to 2.65.0 once the bug is fixed. 'apache-beam<2.65.0', 'conllu', 'mlcroissant>=1.0.9', @@ -150,6 +149,9 @@ # nltk==3.8.2 is broken: https://github.com/nltk/nltk/issues/3293 'nltk==3.8.1', 'tldextract', + # tensorflow==2.20.0 is not compatible with gcld3 because of protobuf + # version conflict. + 'tensorflow<2.20.0', ], 'c4_wsrs': ['apache-beam<2.65.0'], 'cats_vs_dogs': ['matplotlib'], @@ -167,11 +169,17 @@ 'scipy', ], 'librispeech': ['pydub'], # and ffmpeg installed - 'lsun': ['tensorflow-io[tensorflow]'], - # sklearn version required to avoid conflict with librosa from - # https://github.com/scikit-learn/scikit-learn/issues/14485 - # See https://github.com/librosa/librosa/issues/1160 - 'nsynth': ['crepe>=0.0.11', 'librosa', 'scikit-learn==0.20.3'], + 'lsun': [ + # tensorflow-io is compiled against specific versions of TF. + 'tensorflow-io[tensorflow]', + ], + 'nsynth': [ + 'crepe', + 'librosa', + # tensorflow==2.20.0 is not compatible with librosa because of protobuf + # version conflict. + 'tensorflow<2.20.0', + ], 'ogbg_molpcba': ['pandas', 'networkx'], 'pet_finder': ['pandas'], 'qm9': ['pandas'], @@ -196,7 +204,7 @@ # Those datasets have dependencies which conflict with the rest of TFDS, so # running them in an isolated environments. -ISOLATED_DATASETS = ('nsynth', 'lsun') +ISOLATED_DATASETS = ('c4', 'lsun', 'nsynth') # Extra dataset deps are required for the tests all_dataset_dependencies = list( @@ -238,18 +246,21 @@ license='Apache 2.0', packages=setuptools.find_packages(), package_data={ - 'tensorflow_datasets': DATASET_FILES + [ - # Bundle `datasets/` folder in PyPI releases - 'datasets/*/*', - 'core/utils/colormap.csv', - 'scripts/documentation/templates/*', - 'url_checksums/*', - 'checksums.tsv', - 'community-datasets.toml', - 'dataset_collections/*/*.md', - 'dataset_collections/*/*.bib', - 'core/valid_tags.txt', - ], + 'tensorflow_datasets': ( + DATASET_FILES + + [ + # Bundle `datasets/` folder in PyPI releases + 'datasets/*/*', + 'core/utils/colormap.csv', + 'scripts/documentation/templates/*', + 'url_checksums/*', + 'checksums.tsv', + 'community-datasets.toml', + 'dataset_collections/*/*.md', + 'dataset_collections/*/*.bib', + 'core/valid_tags.txt', + ] + ), }, exclude_package_data={ 'tensorflow_datasets': [ diff --git a/tensorflow_datasets/core/lazy_imports_lib_test.py b/tensorflow_datasets/core/lazy_imports_lib_test.py index 13e2ecbc5fd..b03700113fc 100644 --- a/tensorflow_datasets/core/lazy_imports_lib_test.py +++ b/tensorflow_datasets/core/lazy_imports_lib_test.py @@ -25,17 +25,18 @@ class LazyImportsTest(testing.TestCase, parameterized.TestCase): # The following deps are not in the test list because the datasets that # require them need to have their tests run in isolation: # * crepe (NSynth) + # * gcld3 (c4) + # * langdetect (c4) # * librosa (NSynth) + # * nltk (c4) + # * tldextract (c4) @parameterized.parameters( "bs4", "cv2", - "gcld3", "gcsfs_store", - "langdetect", "lxml", "matplotlib", "mwparserfromhell", - "nltk", "os", "pandas", "pretty_midi", @@ -44,7 +45,6 @@ class LazyImportsTest(testing.TestCase, parameterized.TestCase): "scipy", "skimage", "tifffile", - "tldextract", "zarr", ) def test_import(self, module_name):