style: PEP8

scandeval/__init__.py

@@ -1,27 +1,26 @@
+from termcolor import colored
+import logging
+import os
+from .benchmark import Benchmark  # noqa
+from .angry_tweets import AngryTweetsBenchmark  # noqa
+from .dane import DaneBenchmark  # noqa
+from .dkhate import DkHateBenchmark  # noqa
+from .europarl1 import Europarl1Benchmark  # noqa
+from .europarl2 import Europarl2Benchmark  # noqa
+from .lcc1 import Lcc1Benchmark  # noqa
+from .lcc2 import Lcc2Benchmark  # noqa
+from .twitter_sent import TwitterSentBenchmark  # noqa
+from .utils import block_terminal_output
+
 __version__ = '0.0.0'  # noqa
 
 # Block unwanted terminal outputs
-from .utils import block_terminal_output
 block_terminal_output()
 
 # Set up logging
-import logging
-from termcolor import colored
 format = colored('%(asctime)s [%(levelname)s] <%(name)s>\n↳ ', 'green') + \
          colored('%(message)s', 'yellow')
 logging.basicConfig(level=logging.INFO, format=format)
 
 # Disable parallelisation when tokenizing, as that can lead to errors
-import os
 os.environ['TOKENIZERS_PARALLELISM'] = 'false'
-
-# Import benchmark classes
-from .benchmark import Benchmark  # noqa
-from .angry_tweets import AngryTweetsBenchmark  # noqa
-from .dane import DaneBenchmark  # noqa
-from .dkhate import DkHateBenchmark  # noqa
-from .europarl1 import Europarl1Benchmark  # noqa
-from .europarl2 import Europarl2Benchmark  # noqa
-from .lcc1 import Lcc1Benchmark  # noqa
-from .lcc2 import Lcc2Benchmark  # noqa
-from .twitter_sent import TwitterSentBenchmark  # noqa

scandeval/base.py

@@ -210,7 +210,6 @@ def _load_model(self,
             return dict(model=model, tokenizer=tokenizer)
 
         elif framework == 'spacy':
-            import spacy
             local_model_id = model_id.split('/')[-1]
 
             # Download the model if it has not already been so
@@ -258,8 +257,9 @@ def _preprocess_data(self,
         pass
 
     @abstractmethod
-    def _load_data_collator(self,
-                           tokenizer: Optional[PreTrainedTokenizerBase] = None):
+    def _load_data_collator(
+            self,
+            tokenizer: Optional[PreTrainedTokenizerBase] = None):
         '''Load the data collator used to prepare samples during finetuning.
 
         Args:
@@ -352,7 +352,7 @@ def _fetch_model_metadata(model_id: str) -> Dict[str, str]:
 
         # Fetch the frameworks from the model website
         frameworks = [a['tag-id'] for a in a_tags_with_class
-                                  if 'tag-red' in a['class']]
+                      if 'tag-red' in a['class']]
 
         # Extract a single valid framework in which the model has been
         # implemented
@@ -366,7 +366,7 @@ def _fetch_model_metadata(model_id: str) -> Dict[str, str]:
 
         # Fetch the model tasks from the model website
         tasks = [a['tag-id'] for a in a_tags_with_class
-                             if 'tag-white' in a['class']]
+                 if 'tag-white' in a['class']]
 
         # Extract a single valid task on which the model has been trained. If
         # no task has been specified on the model card then assume that it is

scandeval/benchmark.py

@@ -141,7 +141,7 @@ def __call__(self,
             model_ids = [model_ids]
 
         if datasets is None:
-            datasets = [dataset for dataset, _, _ in  self._benchmarks]
+            datasets = [dataset for dataset, _, _ in self._benchmarks]
 
         benchmarks = [(dataset, alias, cls)
                       for dataset, alias, cls in self._benchmarks

scandeval/cli.py

@@ -24,7 +24,7 @@
               help='The name of the benchmark dataset. If not specified then '
                    'all datasets will be benchmarked.')
 @click.option('--language', '-l',
-              default=['da' ,'sv', 'no'],
+              default=['da', 'sv', 'no'],
               show_default=True,
               multiple=True,
               type=click.Choice(['da', 'sv', 'no']),

scandeval/dane.py

@@ -80,7 +80,7 @@ def _load_data(self) -> Tuple[Dataset, Dataset]:
     @staticmethod
     def _remove_misc_tags(examples: dict) -> dict:
         examples['orig_labels'] = [['O' if label[-4:] == 'MISC' else label
-                                        for label in label_list]
+                                    for label in label_list]
                                    for label_list in examples['orig_labels']]
         return examples
 
@@ -106,13 +106,13 @@ def _compute_metrics(self,
 
             # Remove ignored index (special tokens)
             predictions = [
-                [id2label[p] for p, l in zip(prediction, label)
-                             if l != -100]
+                [id2label[pred] for pred, lbl in zip(prediction, label)
+                 if lbl != -100]
                 for prediction, label in zip(raw_predictions, labels)
             ]
             labels = [
-                [id2label[l] for _, l in zip(prediction, label)
-                             if l != -100]
+                [id2label[lbl] for _, lbl in zip(prediction, label)
+                 if lbl != -100]
                 for prediction, label in zip(raw_predictions, labels)
             ]
 

scandeval/europarl1.py

@@ -92,11 +92,12 @@ def _log_metrics(self,
         test_std_err *= 100
 
         if not np.isnan(train_std_err):
-            msg = (f'Mean macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Mean macro-average F1-scores on Europarl1 for '
+                   f'{model_id}:\n'
                    f'  - Train: {train_mean:.2f} +- {train_std_err:.2f}\n'
                    f'  - Test: {test_mean:.2f} +- {test_std_err:.2f}')
         else:
-            msg = (f'Macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Macro-average F1-scores on Europarl1 for {model_id}:\n'
                    f'  - Train: {train_mean:.2f}\n'
                    f'  - Test: {test_mean:.2f}')
 

scandeval/europarl2.py

@@ -92,11 +92,12 @@ def _log_metrics(self,
         test_std_err *= 100
 
         if not np.isnan(train_std_err):
-            msg = (f'Mean macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Mean macro-average F1-scores on Europarl2 for '
+                   f'{model_id}:\n'
                    f'  - Train: {train_mean:.2f} +- {train_std_err:.2f}\n'
                    f'  - Test: {test_mean:.2f} +- {test_std_err:.2f}')
         else:
-            msg = (f'Macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Macro-average F1-scores on Europarl2 for {model_id}:\n'
                    f'  - Train: {train_mean:.2f}\n'
                    f'  - Test: {test_mean:.2f}')
 

scandeval/lcc1.py

@@ -92,11 +92,11 @@ def _log_metrics(self,
         test_std_err *= 100
 
         if not np.isnan(train_std_err):
-            msg = (f'Mean macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Mean macro-average F1-scores on LCC1 for {model_id}:\n'
                    f'  - Train: {train_mean:.2f} +- {train_std_err:.2f}\n'
                    f'  - Test: {test_mean:.2f} +- {test_std_err:.2f}')
         else:
-            msg = (f'Macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Macro-average F1-scores on LCC1 for {model_id}:\n'
                    f'  - Train: {train_mean:.2f}\n'
                    f'  - Test: {test_mean:.2f}')
 

scandeval/lcc2.py

@@ -92,11 +92,11 @@ def _log_metrics(self,
         test_std_err *= 100
 
         if not np.isnan(train_std_err):
-            msg = (f'Mean macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Mean macro-average F1-scores on LCC2 for {model_id}:\n'
                    f'  - Train: {train_mean:.2f} +- {train_std_err:.2f}\n'
                    f'  - Test: {test_mean:.2f} +- {test_std_err:.2f}')
         else:
-            msg = (f'Macro-average F1-scores on AngryTweets for {model_id}:\n'
+            msg = (f'Macro-average F1-scores on LCC2 for {model_id}:\n'
                    f'  - Train: {train_mean:.2f}\n'
                    f'  - Test: {test_mean:.2f}')
 

scandeval/text_classification.py

@@ -64,8 +64,9 @@ def __init__(self,
                          batch_size=batch_size,
                          verbose=verbose)
 
-    def _load_data_collator(self,
-                           tokenizer: Optional[PreTrainedTokenizerBase] = None):
+    def _load_data_collator(
+            self,
+            tokenizer: Optional[PreTrainedTokenizerBase] = None):
         '''Load the data collator used to prepare samples during finetuning.
 
         Args:
@@ -107,14 +108,15 @@ def _preprocess_data(self,
         '''
         if framework in ['pytorch', 'tensorflow', 'jax']:
             tokenizer = kwargs['tokenizer']
-            map_fn = lambda examples: tokenizer(examples['doc'],
-                                                truncation=True,
-                                                padding=True)
-            tokenised = dataset.map(map_fn, batched=True)
-
-            map_fn = partial(self.create_numerical_labels,
-                             label2id=kwargs['config'].label2id)
-            preprocessed = tokenised.map(map_fn, batched=True)
+
+            def tokenise(examples: dict) -> dict:
+                doc = examples['doc']
+                return tokenizer(doc, truncation=True, padding=True)
+            tokenised = dataset.map(tokenise, batched=True)
+
+            numericalise = partial(self.create_numerical_labels,
+                                   label2id=kwargs['config'].label2id)
+            preprocessed = tokenised.map(numericalise, batched=True)
 
             return preprocessed.remove_columns(['doc', 'orig_label'])
 

scandeval/token_classification.py

@@ -153,8 +153,9 @@ def _preprocess_data(self,
         elif framework == 'spacy':
             return dataset.map(self._collect_docs, batched=True)
 
-    def _load_data_collator(self,
-                           tokenizer: Optional[PreTrainedTokenizerBase] = None):
+    def _load_data_collator(
+            self,
+            tokenizer: Optional[PreTrainedTokenizerBase] = None):
         '''Load the data collator used to prepare samples during finetuning.
 
         Args:

scandeval/twitter_sent.py

@@ -92,7 +92,7 @@ def _log_metrics(self,
         test_std_err *= 100
 
         if not np.isnan(train_std_err):
-            msg = (f'Mean macro-average F1-scores on TwitterSent for {model_id}:\n'
+            msg = (f'Mean macro-average F1-scores on TwitterSent {model_id}:\n'
                    f'  - Train: {train_mean:.2f} +- {train_std_err:.2f}\n'
                    f'  - Test: {test_mean:.2f} +- {test_std_err:.2f}')
         else:

scandeval/utils.py

@@ -16,14 +16,13 @@
                           FlaxAutoModelForSequenceClassification)
 
 
-MODEL_CLASSES = {
-    'pytorch': {'token-classification': AutoModelForTokenClassification,
-                'text-classification': AutoModelForSequenceClassification},
-    'tensorflow': {'token-classification': TFAutoModelForTokenClassification,
-                   'text-classification': TFAutoModelForSequenceClassification},
-    'jax': {'token-classification': FlaxAutoModelForTokenClassification,
-            'text-classification': FlaxAutoModelForSequenceClassification}
-}
+PT_CLS = {'token-classification': AutoModelForTokenClassification,
+          'text-classification': AutoModelForSequenceClassification}
+TF_CLS = {'token-classification': TFAutoModelForTokenClassification,
+          'text-classification': TFAutoModelForSequenceClassification}
+JAX_CLS = {'token-classification': FlaxAutoModelForTokenClassification,
+           'text-classification': FlaxAutoModelForSequenceClassification}
+MODEL_CLASSES = dict(pytorch=PT_CLS, tensorflow=TF_CLS, jax=JAX_CLS)
 
 
 class InvalidBenchmark(Exception):
@@ -99,7 +98,8 @@ def f(*args, **kwargs):
     def get_no_inst(self, cls):
         for parent in cls.__mro__[1:]:
             overridden = getattr(parent, self.name, None)
-            if overridden: break
+            if overridden:
+                break
 
         @wraps(self.mthd, assigned=('__name__', '__module__'))
         def f(*args, **kwargs):
@@ -113,4 +113,5 @@ def use_parent_doc(self, func, source):
         func.__doc__ = source.__doc__
         return func
 
+
 doc_inherit = DocInherit