Ability to filter available testsets by language pair (#211)

* Ability to filter available testsets by language pair
   Using

       sacrebleu --list -l en-fr

   test sets available in English-French are listed.

* Remove Python 3.6 from testing, added Python 3.10

Co-authored-by: Matt Post <mattpost@microsoft.com>

.github/workflows/check-build.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
         exclude:
           - os: windows-latest
             python-version: '3.6'   # test fails due to UTF8 stuff

CHANGELOG.md

@@ -1,13 +1,18 @@
 # Release Notes
 
-- 2.2.2 (2022-09-29)
+- 2.2.2 (2022-XX-XX)
   Features:
-  - Added `-tok flores101` and `-tok flores200`, a.k.a. `spbleu`.
+  - (#203) Added `-tok flores101` and `-tok flores200`, a.k.a. `spbleu`.
     These are multilingual tokenizations that make use of the
     multilingual SPM models released by Facebook and described in the
     following papers:
     * Flores-101: https://arxiv.org/abs/2106.03193
     * Flores-200: https://arxiv.org/abs/2207.04672
+  - (#211) You can now list all test sets for a language pair with `--list SRC-TRG`.
+    Thanks to Jaume Zaragoza (@ZJaume) for adding this feature.
+
+  Changes:
+  - Removed testing support for Python 3.6 (end-of-lifed: https://endoflife.date/python).
 
 - 2.2.1 (2022-09-13)
   Bugfix: Standard usage was returning (and using) each reference twice.

README.md

@@ -67,7 +67,8 @@ following command instead, to perform a full installation with dependencies:
 # Command-line Usage
 
 You can get a list of available test sets with `sacrebleu --list`. Please see [DATASETS.md](DATASETS.md)
-for an up-to-date list of supported datasets.
+for an up-to-date list of supported datasets. You can also list available test sets for a given language pair
+with `sacrebleu --list -l en-fr`.
 
 ## Basics
 

sacrebleu/sacrebleu.py

@@ -42,7 +42,7 @@
 from .utils import smart_open, filter_subset, get_langpairs_for_testset, get_available_testsets
 from .utils import print_test_set, print_subset_results, get_reference_files, download_test_set
 from .utils import args_to_dict, sanity_check_lengths, print_results_table, print_single_results
-from .utils import Color
+from .utils import get_available_testsets_for_langpair, Color
 
 from . import __version__ as VERSION
 
@@ -247,8 +247,13 @@ def main():
                 fields = DATASETS[args.test_set].fieldnames(pair)
                 print(f'{pair}: {", ".join(fields)}')
         else:
-            print('The available test sets are:')
-            for testset in sorted(get_available_testsets()):
+            if args.langpair:
+                print(f'The available test sets for {args.langpair} are:')
+                testsets = get_available_testsets_for_langpair(args.langpair)
+            else:
+                print('The available test sets are:')
+                testsets = get_available_testsets()
+            for testset in sorted(testsets):
                 desc = DATASETS[testset].description.strip()
                 print(f'{testset:<30}: {desc}')
         sys.exit(0)
@@ -292,7 +297,7 @@ def main():
         sacrelogger.error('I need exactly one of (a) a predefined test set (-t) or (b) a list of references')
         sys.exit(1)
     elif args.langpair is None:
-        sacrelogger.error('I need a language pair (-l).')
+        sacrelogger.error('I need a language pair (-l). Use --list to see available language pairs for this test set.')
         sys.exit(1)
     else:
         for test_set in args.test_set.split(','):

sacrebleu/utils.py

@@ -456,6 +456,20 @@ def get_available_testsets() -> List[str]:
     """Return a list of available test sets."""
     return sorted(DATASETS.keys(), reverse=True)
 
+def get_available_testsets_for_langpair(langpair: str) -> List[str]:
+    """Return a list of available test sets for a given language pair"""
+    parts = langpair.split('-')
+    srclang = parts[0]
+    trglang = parts[1]
+
+    testsets = []
+    for dataset in DATASETS.values():
+        if f'{srclang}-{trglang}' in dataset.langpairs \
+                or f'{trglang}-{srclang}' in dataset.langpairs:
+            testsets.append(dataset.name)
+
+    return testsets
+
 
 def get_available_origlangs(test_sets, langpair) -> List[str]:
     """Return a list of origlang values in according to the raw SGM files."""

test/test_api.py

@@ -13,7 +13,7 @@
 
 import pytest
 
-from sacrebleu.utils import get_available_testsets, get_langpairs_for_testset
+from sacrebleu.utils import get_available_testsets, get_available_testsets_for_langpair, get_langpairs_for_testset
 from sacrebleu.utils import get_source_file, get_reference_files
 from sacrebleu.dataset import DATASETS
 
@@ -53,6 +53,23 @@ def test_api_get_available_testsets():
         assert "slashdot_" + testset not in available
 
 
+def test_api_get_available_testsets_for_langpair():
+    """
+    Loop over the datasets directly, and ensure the API function returns
+    the test sets found.
+    """
+    available = get_available_testsets_for_langpair('en-it')
+    assert type(available) is list
+    assert "wmt09" in available
+    assert "wmt15" not in available
+
+    available = get_available_testsets_for_langpair('en-fr')
+    assert type(available) is list
+    assert "wmt11" in available
+    assert "mtedx/test" in available
+    assert "wmt20" not in available
+
+
 def test_api_get_langpairs_for_testset():
     """
     Loop over the datasets directly, and ensure the API function