From 7ac1dc74335b8935e4ac897e8d92d9c563fdf110 Mon Sep 17 00:00:00 2001 From: vycezhong Date: Thu, 21 Jan 2021 16:12:46 +0800 Subject: [PATCH] fix init --- byteps/torch/__init__.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/byteps/torch/__init__.py b/byteps/torch/__init__.py index dd3729aaa..73fe213f6 100644 --- a/byteps/torch/__init__.py +++ b/byteps/torch/__init__.py @@ -55,7 +55,8 @@ def __init__(self, params, named_parameters, compression, 'tuples (name, parameter), usually produced by ' 'model.named_parameters().') - dups = _DistributedOptimizer.find_duplicates([k for k, _ in named_parameters]) + dups = _DistributedOptimizer.find_duplicates( + [k for k, _ in named_parameters]) if len(dups) > 0: raise ValueError('Parameter names in named_parameters must be unique. ' 'Found duplicates: %s' % ', '.join(dups)) @@ -70,7 +71,8 @@ def __init__(self, params, named_parameters, compression, # https://github.com/pytorch/pytorch/issues/7733 self._parameter_names = {v.__hash__(): k for k, v in sorted(named_parameters)} - self._tensor_list = [tensor for name, tensor in named_parameters] + self._tensor_list = [tensor for name, + tensor in named_parameters] else: self._is_tensor_instance = False self._parameter_names = {v: k for k, v @@ -134,7 +136,8 @@ def _push_pull_grad_async(self, p): else: tensor = p.grad tensor_compressed, ctx = self._compression.compress(tensor) - handle = byteps_push_pull(tensor_compressed, average=True, name="Gradient."+name) + handle = byteps_push_pull( + tensor_compressed, average=True, name="Gradient."+name) return handle, ctx def _make_hook(self, p): @@ -176,7 +179,8 @@ def synchronize(self): @contextmanager def skip_synchronize(self): if self._enable_async: - raise AssertionError("skip_synchronize cannot be used in async training") + raise AssertionError( + "skip_synchronize cannot be used in async training") self._should_sync = False try: yield @@ -201,7 +205,8 @@ def step(self, closure=None): name = self._parameter_names.get(p.__hash__()) else: name = self._parameter_names.get(p) - handle = byteps_push_pull(p, average=False, name="AsyncParam."+name) + handle = byteps_push_pull( + p, average=False, name="Parameter."+name) _, ctx = self._compression.compress(p) self._handles[p] = (handle, ctx) @@ -378,7 +383,8 @@ def _from_tensor(): key = '%s.%d' % (option_key, index) dtypes = _get_types(option_value) option_tensor = torch.Tensor([option_value]).cuda() - callbacks[key] = _create_option_callback(index, option_key, option_tensor, dtypes) + callbacks[key] = _create_option_callback( + index, option_key, option_tensor, dtypes) params.append((key, option_tensor)) # The params list here is ordered by the layers in the model