From 8ba90e59b172b95687371477ce69503b1e0dee55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 22 Jul 2021 11:01:07 +0200 Subject: [PATCH 1/7] clarify closure usage in gan example --- docs/source/common/optimizers.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index 804bf2b1d991a..c0a1f5647fcec 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -398,6 +398,8 @@ For example, here step optimizer A every batch and optimizer B every 2 batches. if (batch_idx + 1) % 2 == 0: # the closure (which includes the `training_step`) won't run if the line below isn't executed optimizer.step(closure=optimizer_closure) + else: + # closure/training_step won't be called in this case # ... # add as many optimizers as you want From 9f325f1756ab000268008658d2558bd5f6fc0af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 22 Jul 2021 12:01:48 +0200 Subject: [PATCH 2/7] Update docs/source/common/optimizers.rst --- docs/source/common/optimizers.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index c0a1f5647fcec..c3cb32d995dad 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -398,8 +398,7 @@ For example, here step optimizer A every batch and optimizer B every 2 batches. if (batch_idx + 1) % 2 == 0: # the closure (which includes the `training_step`) won't run if the line below isn't executed optimizer.step(closure=optimizer_closure) - else: - # closure/training_step won't be called in this case + # ... # add as many optimizers as you want From 4aca1e7967f4668d8e29322ed1d7809f62476c9d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 22 Jul 2021 10:02:51 +0000 Subject: [PATCH 3/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/common/optimizers.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index c3cb32d995dad..f6e7845875338 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -398,7 +398,7 @@ For example, here step optimizer A every batch and optimizer B every 2 batches. if (batch_idx + 1) % 2 == 0: # the closure (which includes the `training_step`) won't run if the line below isn't executed optimizer.step(closure=optimizer_closure) - + # ... # add as many optimizers as you want From 48d03edf71d7c9eecefdd6dfdef500fcf0dc2653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Thu, 22 Jul 2021 12:04:00 +0200 Subject: [PATCH 4/7] remove empty line --- docs/source/common/optimizers.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index f6e7845875338..804bf2b1d991a 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -399,7 +399,6 @@ For example, here step optimizer A every batch and optimizer B every 2 batches. # the closure (which includes the `training_step`) won't run if the line below isn't executed optimizer.step(closure=optimizer_closure) - # ... # add as many optimizers as you want From 45c9eaa5efa00118afb7b20d79d1315a59b5e1ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 26 Jul 2021 10:04:08 +0200 Subject: [PATCH 5/7] update closure description --- docs/source/common/optimizers.rst | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index 804bf2b1d991a..dd2e0357d0e00 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -299,10 +299,14 @@ Under the hood, Lightning does the following: for epoch in epochs: for batch in data: - loss = model.training_step(batch, batch_idx, ...) - optimizer.zero_grad() - loss.backward() - optimizer.step() + + def closure(): + loss = model.training_step(batch, batch_idx, ...) + optimizer.zero_grad() + loss.backward() + return loss + + optimizer.step(closure) for lr_scheduler in lr_schedulers: lr_scheduler.step() @@ -314,14 +318,22 @@ In the case of multiple optimizers, Lightning does the following: for epoch in epochs: for batch in data: for opt in optimizers: - loss = model.training_step(batch, batch_idx, optimizer_idx) - opt.zero_grad() - loss.backward() - opt.step() + + def closure(): + loss = model.training_step(batch, batch_idx, optimizer_idx) + opt.zero_grad() + loss.backward() + return loss + + opt.step(closure) for lr_scheduler in lr_schedulers: lr_scheduler.step() +As can be seen in the code snippet above, Lightning defines a closure with ``training_step``, ``zero_grad`` +and ``backward`` for the optimizer to execute. This mechanism is in place to support optimizers which operate on the +output of the closure (e.g. the loss) or need to call the closure several times (e.g. :class:`~torch.optim.LBFGS`). + .. warning:: Before 1.2.2, Lightning internally calls ``backward``, ``step`` and ``zero_grad`` in the order. From 1.2.2, the order is changed to ``zero_grad``, ``backward`` and ``step``. From b61f978cf196dc806ba7c1dc86a1fe72727c154c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 08:16:09 +0000 Subject: [PATCH 6/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/source/governance.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/governance.rst b/docs/source/governance.rst index 5c29f7d0da544..4114ccdb8a818 100644 --- a/docs/source/governance.rst +++ b/docs/source/governance.rst @@ -39,7 +39,7 @@ Board Alumni ------ -- Jeff Yang (`ydcjeff `_) +- Jeff Yang (`ydcjeff `_) - Jeff Ling (`jeffling `_) - Teddy Koker (`teddykoker `_) - Nate Raw (`nateraw `_) From 5762b1497ca06767c3505d935b5bd7d118c395c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20W=C3=A4lchli?= Date: Mon, 26 Jul 2021 10:24:32 +0200 Subject: [PATCH 7/7] add manual closure call --- docs/source/common/optimizers.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/common/optimizers.rst b/docs/source/common/optimizers.rst index dd2e0357d0e00..7bd72730ac4ce 100644 --- a/docs/source/common/optimizers.rst +++ b/docs/source/common/optimizers.rst @@ -408,8 +408,11 @@ For example, here step optimizer A every batch and optimizer B every 2 batches. # update discriminator every 2 steps if optimizer_idx == 1: if (batch_idx + 1) % 2 == 0: - # the closure (which includes the `training_step`) won't run if the line below isn't executed + # the closure (which includes the `training_step`) will be executed by `optimizer.step` optimizer.step(closure=optimizer_closure) + else: + # optional: call the closure by itself to run `training_step` + `backward` without an optimizer step + optimizer_closure() # ... # add as many optimizers as you want