Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[DDP] Rename num_iterations -> num_forward_calls #103283

Closed
wants to merge 1 commit into from
Closed

[DDP] Rename num_iterations -> num_forward_calls #103283

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 5 additions & 5 deletions torch/nn/parallel/distributed.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ def forward(ctx, reducer, ddp_state, *inputs):
def backward(ctx, *grad_outputs):
# Enqueue delay allreduce for static graph training on the first
# iteration.
if ctx.ddp_state["static_graph"] and ctx.ddp_state["num_iterations"] == 1:
if ctx.ddp_state["static_graph"] and ctx.ddp_state["num_forward_calls"] == 1:
Variable._execution_engine.queue_callback( # type: ignore[call-arg,misc]
ctx.reducer._delay_all_reduce
)
Expand Down Expand Up @@ -1047,7 +1047,7 @@ def _ddp_init_helper(
(4) Logging construction-time DDP logging data
(5) passing a handle of DDP to SyncBatchNorm Layer
"""
self.num_iterations = 0
self.num_forward_calls = 0
# Notice, the parameters order is not in the order in which they are used,
# especially in models with control flow.
#
Expand Down Expand Up @@ -1381,7 +1381,7 @@ def _pre_forward(self, *inputs, **kwargs):
if torch.is_grad_enabled() and self.require_backward_grad_sync:
assert self.logger is not None
self.logger.set_runtime_stats_and_log()
self.num_iterations += 1
self.num_forward_calls += 1
self.reducer.prepare_for_forward()

# Notify the join context that this process has not joined, if
Expand Down Expand Up @@ -1466,11 +1466,11 @@ def _post_forward(self, output):
# TODO: DDPSink is currently enabled for unused parameter detection and
# static graph training for first iteration.
if (self.find_unused_parameters and not self.static_graph) or (
self.static_graph and self.num_iterations == 1
self.static_graph and self.num_forward_calls == 1
):
ddp_state = {
"static_graph": self.static_graph,
"num_iterations": self.num_iterations,
"num_forward_calls": self.num_forward_calls,
}

(
Expand Down