remove multiply_grads

scripts/question_answering/run_squad.py

@@ -23,7 +23,7 @@
 from gluonnlp.models import get_backbone
 from gluonnlp.utils.misc import grouper, repeat, set_seed, parse_ctx, logging_config, count_parameters
 from gluonnlp.initializer import TruncNorm
-from gluonnlp.utils.parameter import clip_grad_global_norm, multiply_grads
+from gluonnlp.utils.parameter import clip_grad_global_norm, grad_global_norm
 
 mx.npx.set_np()
 
@@ -577,9 +577,9 @@ def train(args):
                 params, args.max_grad_norm * num_samples_per_update / loss_denom)
             total_norm = total_norm / (num_samples_per_update / loss_denom)
         else:
-            total_norm, is_finite = multiply_grads(params, loss_denom / num_samples_per_update)
+            total_norm = grad_global_norm(parameters)
 
-        trainer.update(num_samples_per_update / loss_denom, ignore_stale_grad=True)
+        trainer.update(num_samples_per_update / loss_denom)
         if args.num_accumulated != 1:
             # set grad to zero for gradient accumulation
             qa_net.collect_params().zero_grad()

src/gluonnlp/utils/parameter.py

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Utility functions for trainer and parameters."""
-__all__ = ['grad_global_norm', 'clip_grad_global_norm', 'multiply_grads']
+__all__ = ['grad_global_norm', 'clip_grad_global_norm']
 
 
 import warnings
@@ -152,37 +152,3 @@ def clip_grad_global_norm(parameters: Iterable[Parameter],
             for arr in p.list_grad():
                 arr *= scale
     return total_norm, ratio, is_finite
-
-
-def multiply_grads(parameters: Iterable[Parameter],
-                   scale: float,
-                   check_isfinite: bool = True) -> Tuple[float]:
-    """
-    Multiplies grads by a constant scale
-
-    Parameters
-    ----------
-    parameters
-        The list of parameters to calculate the norm
-    scale
-        The normalize multiplier to normalize the gradient
-    Returns
-    -------
-    total_norm
-        The total norm
-    is_finite
-        Whether the total norm is finite
-    """
-    total_norm = grad_global_norm(parameters)
-    is_finite = bool(np.isfinite(total_norm))
-    if check_isfinite and not is_finite:
-        warnings.warn(
-            UserWarning('nan or inf is detected. Clipping results will be undefined.'
-                        ' Thus, skip clipping'),
-            stacklevel=2)
-        return total_norm, is_finite
-    for p in parameters:
-        if p.grad_req != 'null':
-            for arr in p.list_grad():
-                arr *= scale
-    return total_norm, is_finite