Merge branch 'master' into vision

allenai · Aug 20, 2020 · 95e8253 · 95e8253
2 parents e74a736 + 87a61ad
commit 95e8253
Show file tree

Hide file tree

Showing 32 changed files with 765 additions and 233 deletions.
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
@@ -51,7 +51,7 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
 
-    - uses: actions/cache@v1
+    - uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
@@ -119,7 +119,7 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
 
-    - uses: actions/cache@v1
+    - uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
@@ -164,7 +164,7 @@ jobs:
       with:
         python-version: 3.7
 
-    - uses: actions/cache@v1
+    - uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
@@ -332,7 +332,7 @@ jobs:
       with:
         python-version: 3.7
 
-    - uses: actions/cache@v1
+    - uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -59,7 +59,7 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
 
-    - uses: actions/cache@v1
+    - uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}
@@ -128,7 +128,7 @@ jobs:
       with:
         python-version: ${{ matrix.python }}
 
-    - uses: actions/cache@v1
+    - uses: actions/cache@v2
       with:
         path: ${{ env.pythonLocation }}
         key: ${{ runner.os }}-pydeps-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ hashFiles('dev-requirements.txt') }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added a workflow to GitHub Actions that will automatically close unassigned stale issues and
   ping the assignees of assigned stale issues.
 
+### Fixed
+
+- Fixed a bug in distributed metrics that caused nan values due to repeated addition of an accumulated variable.
+
 ## [v1.1.0rc3](https://github.com/allenai/allennlp/releases/tag/v1.1.0rc3) - 2020-08-12
 
 ### Fixed
@@ -34,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Added a `py.typed` marker. Fixed type annotations in `allennlp.training.util`.
 - Fixed problem with automatically detecting whether tokenization is necessary.
   This affected primarily the Roberta SST model.
+- Improved help text for using the --overrides command line flag.
 
 ## [v1.1.0rc2](https://github.com/allenai/allennlp/releases/tag/v1.1.0rc2) - 2020-07-31
 

diff --git a/allennlp/commands/evaluate.py b/allennlp/commands/evaluate.py
@@ -53,7 +53,11 @@ def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.Argument
             "--overrides",
             type=str,
             default="",
-            help="a JSON structure used to override the experiment configuration",
+            help=(
+                "a json(net) structure used to override the experiment configuration, e.g., "
+                "'{\"iterator.batch_size\": 16}'.  Nested parameters can be specified either"
+                " with nested dictionaries or with dot syntax."
+            ),
         )
 
         subparser.add_argument(

diff --git a/allennlp/commands/find_learning_rate.py b/allennlp/commands/find_learning_rate.py
@@ -53,7 +53,11 @@ def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.Argument
             "--overrides",
             type=str,
             default="",
-            help="a JSON structure used to override the experiment configuration",
+            help=(
+                "a json(net) structure used to override the experiment configuration, e.g., "
+                "'{\"iterator.batch_size\": 16}'.  Nested parameters can be specified either"
+                " with nested dictionaries or with dot syntax."
+            ),
         )
         subparser.add_argument(
             "--start-lr", type=float, default=1e-5, help="learning rate to start the search"

diff --git a/allennlp/commands/predict.py b/allennlp/commands/predict.py
@@ -78,7 +78,11 @@ def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.Argument
             "--overrides",
             type=str,
             default="",
-            help="a JSON structure used to override the experiment configuration",
+            help=(
+                "a json(net) structure used to override the experiment configuration, e.g., "
+                "'{\"iterator.batch_size\": 16}'.  Nested parameters can be specified either"
+                " with nested dictionaries or with dot syntax."
+            ),
         )
 
         subparser.add_argument(

diff --git a/allennlp/commands/train.py b/allennlp/commands/train.py
@@ -71,7 +71,11 @@ def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.Argument
             "--overrides",
             type=str,
             default="",
-            help="a JSON structure used to override the experiment configuration",
+            help=(
+                "a json(net) structure used to override the experiment configuration, e.g., "
+                "'{\"iterator.batch_size\": 16}'.  Nested parameters can be specified either"
+                " with nested dictionaries or with dot syntax."
+            ),
         )
 
         subparser.add_argument(
@@ -81,8 +85,10 @@ def add_subparser(self, parser: argparse._SubParsersAction) -> argparse.Argument
         subparser.add_argument(
             "--dry-run",
             action="store_true",
-            help="do not train a model, but create a vocabulary, show dataset statistics and "
-            "other training information",
+            help=(
+                "do not train a model, but create a vocabulary, show dataset statistics and "
+                "other training information"
+            ),
         )
         subparser.add_argument(
             "--file-friendly-logging",
@@ -256,7 +262,7 @@ def train_model(
         world_size = num_nodes * num_procs
 
         logging.info(
-            f"Switching to distributed training mode since multiple GPUs are configured | "
+            "Switching to distributed training mode since multiple GPUs are configured | "
             f"Master is at: {master_addr}:{master_port} | Rank of this node: {node_rank} | "
             f"Number of workers in this node: {num_procs} | Number of nodes: {num_nodes} | "
             f"World size: {world_size}"

diff --git a/allennlp/training/metrics/attachment_scores.py b/allennlp/training/metrics/attachment_scores.py
@@ -80,27 +80,27 @@ def __call__(  # type: ignore
         correct_labels = predicted_labels.eq(gold_labels).long() * mask
         correct_labels_and_indices = correct_indices * correct_labels
         labeled_exact_match = (correct_labels_and_indices + ~mask).prod(dim=-1)
+        total_sentences = correct_indices.size(0)
+        total_words = correct_indices.numel() - (~mask).sum()
 
         if is_distributed():
             dist.all_reduce(correct_indices, op=dist.ReduceOp.SUM)
             dist.all_reduce(unlabeled_exact_match, op=dist.ReduceOp.SUM)
             dist.all_reduce(correct_labels_and_indices, op=dist.ReduceOp.SUM)
             dist.all_reduce(labeled_exact_match, op=dist.ReduceOp.SUM)
+            total_sentences = torch.tensor(total_sentences).to(device)
+            total_words = torch.tensor(total_words).to(device)
+            dist.all_reduce(total_sentences, op=dist.ReduceOp.SUM)
+            dist.all_reduce(total_words, op=dist.ReduceOp.SUM)
+            total_sentences = total_sentences.item()
+            total_words = total_words.item()
 
         self._unlabeled_correct += correct_indices.sum()
         self._exact_unlabeled_correct += unlabeled_exact_match.sum()
         self._labeled_correct += correct_labels_and_indices.sum()
         self._exact_labeled_correct += labeled_exact_match.sum()
-        self._total_sentences += correct_indices.size(0)
-        self._total_words += correct_indices.numel() - (~mask).sum()
-
-        if is_distributed():
-            _total_sentences = torch.tensor(self._total_sentences).to(device)
-            _total_words = torch.tensor(self._total_words).to(device)
-            dist.all_reduce(_total_sentences, op=dist.ReduceOp.SUM)
-            dist.all_reduce(_total_words, op=dist.ReduceOp.SUM)
-            self._total_sentences = _total_sentences.item()
-            self._total_words = _total_words.item()
+        self._total_sentences += total_sentences
+        self._total_words += total_words
 
     def get_metric(
         self, reset: bool = False, cuda_device: Union[int, torch.device] = torch.device("cpu"),

diff --git a/allennlp/training/metrics/average.py b/allennlp/training/metrics/average.py
@@ -28,16 +28,18 @@ def __call__(self, value):
         value : `float`
             The value to average.
         """
-        self._total_value += list(self.detach_tensors(value))[0]
-        self._count += 1
+        _total_value = list(self.detach_tensors(value))[0]
+        _count = 1
         if is_distributed():
             device = torch.device("cpu")
-            _count = torch.tensor(self._count).to(device)
-            _total_value = torch.tensor(self._total_value).to(device)
-            dist.all_reduce(_count, op=dist.ReduceOp.SUM)
-            dist.all_reduce(_total_value, op=dist.ReduceOp.SUM)
-            self._count = _count.item()
-            self._total_value = _total_value.item()
+            count = torch.tensor(_count).to(device)
+            total_value = torch.tensor(_total_value).to(device)
+            dist.all_reduce(count, op=dist.ReduceOp.SUM)
+            dist.all_reduce(total_value, op=dist.ReduceOp.SUM)
+            _count = count.item()
+            _total_value = total_value.item()
+        self._count += _count
+        self._total_value += _total_value
 
     @overrides
     def get_metric(self, reset: bool = False):

diff --git a/allennlp/training/metrics/bleu.py b/allennlp/training/metrics/bleu.py
@@ -133,26 +133,32 @@ def __call__(
                 dist.all_reduce(_precision_totals, op=dist.ReduceOp.SUM)
                 precision_matches = _precision_matches.item() / world_size
                 precision_totals = _precision_totals.item() / world_size
+
             self._precision_matches[ngram_size] += precision_matches
             self._precision_totals[ngram_size] += precision_totals
+
         if not self._exclude_indices:
-            self._prediction_lengths += predictions.size(0) * predictions.size(1)
-            self._reference_lengths += gold_targets.size(0) * gold_targets.size(1)
+            _prediction_lengths = predictions.size(0) * predictions.size(1)
+            _reference_lengths = gold_targets.size(0) * gold_targets.size(1)
+
         else:
             from allennlp.training.util import get_valid_tokens_mask
 
             valid_predictions_mask = get_valid_tokens_mask(predictions, self._exclude_indices)
-            self._prediction_lengths += valid_predictions_mask.sum().item()
             valid_gold_targets_mask = get_valid_tokens_mask(gold_targets, self._exclude_indices)
-            self._reference_lengths += valid_gold_targets_mask.sum().item()
+            _prediction_lengths = valid_predictions_mask.sum().item()
+            _reference_lengths = valid_gold_targets_mask.sum().item()
 
         if is_distributed():
-            _prediction_lengths = torch.tensor(self._prediction_lengths).to(device)
-            _reference_lengths = torch.tensor(self._reference_lengths).to(device)
-            dist.all_reduce(_prediction_lengths, op=dist.ReduceOp.SUM)
-            dist.all_reduce(_reference_lengths, op=dist.ReduceOp.SUM)
-            self._prediction_lengths = _prediction_lengths.item()
-            self._reference_lengths = _reference_lengths.item()
+            prediction_lengths = torch.tensor(_prediction_lengths).to(device)
+            reference_lengths = torch.tensor(_reference_lengths).to(device)
+            dist.all_reduce(prediction_lengths, op=dist.ReduceOp.SUM)
+            dist.all_reduce(reference_lengths, op=dist.ReduceOp.SUM)
+            _prediction_lengths = prediction_lengths.item()
+            _reference_lengths = reference_lengths.item()
+
+        self._prediction_lengths += _prediction_lengths
+        self._reference_lengths += _reference_lengths
 
     @overrides
     def get_metric(self, reset: bool = False) -> Dict[str, float]:

diff --git a/allennlp/training/metrics/boolean_accuracy.py b/allennlp/training/metrics/boolean_accuracy.py
@@ -84,17 +84,14 @@ def __call__(
 
         # Since masked positions are correct, we need to explicitly exclude instance predictions
         # where the entire prediction is masked (because they look "correct").
-        correct_count_diff = (correct * keep).sum()
-        total_count_diff = keep.sum()
+        _correct_count = (correct * keep).sum()
+        _total_count = keep.sum()
 
         if is_distributed():
-            dist.all_reduce(correct_count_diff, op=dist.ReduceOp.SUM)
-            dist.all_reduce(total_count_diff, op=dist.ReduceOp.SUM)
-            self._correct_count += correct_count_diff.item()
-            self._total_count += total_count_diff.item()
-        else:
-            self._correct_count += correct_count_diff
-            self._total_count += total_count_diff
+            dist.all_reduce(_correct_count, op=dist.ReduceOp.SUM)
+            dist.all_reduce(_total_count, op=dist.ReduceOp.SUM)
+        self._correct_count += _correct_count.item()
+        self._total_count += _total_count.item()
 
     def get_metric(self, reset: bool = False):
         """

diff --git a/allennlp/training/metrics/categorical_accuracy.py b/allennlp/training/metrics/categorical_accuracy.py
@@ -93,19 +93,17 @@ def __call__(
 
         if mask is not None:
             correct *= mask.view(-1, 1)
-            total_count_diff = mask.sum()
+            _total_count = mask.sum()
         else:
-            total_count_diff = torch.tensor(gold_labels.numel())
-        correct_count_diff = correct.sum()
+            _total_count = torch.tensor(gold_labels.numel())
+        _correct_count = correct.sum()
 
         if is_distributed():
-            dist.all_reduce(correct_count_diff, op=dist.ReduceOp.SUM)
-            dist.all_reduce(total_count_diff, op=dist.ReduceOp.SUM)
-            self.correct_count += correct_count_diff.item()
-            self.total_count += total_count_diff.item()
-        else:
-            self.correct_count += correct_count_diff
-            self.total_count += total_count_diff
+            dist.all_reduce(_correct_count, op=dist.ReduceOp.SUM)
+            dist.all_reduce(_total_count, op=dist.ReduceOp.SUM)
+
+        self.correct_count += _correct_count.item()
+        self.total_count += _total_count.item()
 
     def get_metric(self, reset: bool = False):
         """